/*--------------------------------------------------------------------*/
/*                                                                    */
/*  Effective Bandwidth (b_eff) Benchmark                             */
/*                                                                    */
/*  Author: Rolf Rabenseifner                                         */
/*          Gerrit Schulz                                             */
/*                                                                    */
/*  Copyright (c) Rolf Rabenseifner, HLRS, University of Stuttgart    */
/*                                                                    */
/*  Further information: www.hlrs.de/mpi/b_eff/                       */
/*                                                                    */
#define Revision "3.6"
/*                                                                    */
#define Date     "May 06, 2002"
/*                                                                    */
/*--------------------------------------------------------------------*/

/* Compilation and execution:

   CRAY T3E:  Prerequisites: using moduls mpt
              Compilation on T3E with 128 MB/processor:   
                cc -o b_eff -D MEMORY_PER_PROCESSOR=128 b_eff.c 
              Compilation on T3E with 512 MB/processor:   
                cc -o b_eff -D MEMORY_PER_PROCESSOR=512 b_eff.c 
              Execution:     export MPI_BUFFER_MAX=4099 
                             mpirun -np <NP> ./b_eff

   SX-4:      Prerequisites: -
              Compilation on SX-4 with 256 MB/processor:
                mpicc -o b_eff -D MEMORY_PER_PROCESSOR=256 b_eff.c -lm
              Execution:     mpirun -np <NP> ./b_eff

   SR8000:    Prerequisites: -
              Compilation on SR8000 with 1 GB/processor: 
                mpicc -O4 -pvec +Op -noparallel -o b_eff -D MEMORY_PER_PROCESSOR=1024 b_eff.c
              Execution:     mpiexec -p multi -N <NODES> -n <NP> ./b_eff

   In the case of troubles one can use the following tests:

     -D SHORT                   : reducing several parameters
     -D TEST                    : reducing several parameters extremely
     -D PRINTLEVEL=1  or 2,3,4,5: prints in each measurement loop
     -D NOworstcyc              : removes the worst-cyclic pattern 
     -D NObestBIsection         : removes the best bi-section pattern
     -D NOworstBIsection        : removes the worst bi-section pattern
     -D NOonePingPongPair       : removes the one-ping-pong-pair pat.
     -D NOacyclic               : removes the acyclic patterns
     -D NOcyclic                : removes the cyclic patterns
     -D NOalltoallv             : removes the MPI_Alltoallv method
     -D NOnonblk                : removes the Irecv-Isend-Waitall method
     -D NOshortlasttwo          :

   Most compiler do not need or allow the blank after "-D". 

   Each MPI process can be a single thread (default) or can be multi-threaded.
   In this case, only the master thread executes the b_eff benchmark.
   All other threads are waiting for the end of the parallel region.
   The code must be compiled with the OpenMP flag of the compiler.
   The number of threads per MPI process has to be defined with the
   environment variable OMP_NUM_THREADS. The OMP_DYNAMIC environment variable
   must be set to FALSE. Example:
    - with ksh, bash...: 
        export OMP_NUM_THREADS=8
        export OMP_DYNAMIC=FALSE
    - with csh, tcsh...: 
        setenv OMP_NUM_THREADS 8
        setenv OMP_DYNAMIC FALSE
   If the OpenMP compiler flag is used, it is assumed that the MPI 
   library provides MPI_Init_thread with at least MPI_THREAD_FUNNELED. 
*/ 

#define HELP \
\
" Effective Bandwidth (b_eff) Benchmark Version 3.6\n\n"\
\
"  Execution: \n\n"\
\
"  mpirun  <mpi options> ./b_eff <runtime options> <result>    or\n"\
"  mpiexec <mpi options> ./b_eff <runtime options> <result>\n\n"\
\
" Runtime options: \n\n"\
\
"  -yrange [<min>:<max>]     sets the vertical range for gnuplot\n"\
"                            default is [0.01:10000]\n\n"\
\
"  -help                     displays this information\n\n"\
\
"  -f <result>               prefix of the files generated by b_eff\n"\
"                            <result>.prot, <result>.short etc.\n"\
"                            default is 'b_eff'\n\n"\
\
"  Output:\n\n"\
\
"   <result>.prot            detailed\n"\
"   <result>.short           short\n"\
"   <result>.sum             overview\n"\
"   <result>.plot            plot data\n"\
"   <result>.gps             gnuplot script\n"\
"   <result>.tex             LaTeX source file\n\n"\
\
" Postprocessing: \n\n"\
\
"   gnuplot <result>.gps\n"\
"   latex <result>.tex\n"\
"   dvips <result>.dvi\n\n"\
\
" See also:  www.hlrs.de/mpi/b_eff/ \n"

#define CAUTION \
\
"\n"\
" CAUTION! Due to performance reasons the two largest messages are shortened.\n"\
" In detail that means: Only the best method is used and only two repetitions are performed.\n"\
" The values are copied to the other methods. This behavior has no effect to the b_eff value,\n"\
" but a comparison between the methods makes no sense. If you need these tables because you want\n"\
" to compare the methods please use the compiler option -D NOshortlasttwo.\n"

#include "mpi.h"

/* --- for printing */ 
#include <stdio.h>
/* --- for pow(), log(), exp(): */
#include <math.h>
/* --- for fast malloc(), time(): */
#include <sys/types.h>
/* --- for fast malloc(): */
#include <malloc.h>
/* --- for time(), ctime(): */
#include <time.h>
/* --- for drand48(), srand48(): */
#include <stdlib.h>
/* --- for uname(): */
#include <sys/utsname.h> 
 
#ifdef _OPENMP
#  include <omp.h>
#endif 

#ifndef MEMORY_PER_PROCESSOR 
  *** ERROR ***  "You have to define the CPP macro MEMORY_PER_PROCESSOR on the C compilation command" 
# define MEMORY_PER_PROCESSOR 128 /*MBytes*/
#endif 

#define REPETITIONS 3
#define MIN_REPETITIONS 2   /* to reduce total execution time if necess.*/
#define MSGLENGTHS 21
#define MAX_FACT_2 12       /* message lengths by step with factor 2
                               from 2**0 until 2**MAX_FACT_2 */
#define PRE_LOOPS 1         /* Ping-pongs before the measurement loop */
#define WITH_PRELOOPS 18    /* PRE_LOOPS are used if i_msg<WITH_PRELOOPS */
#define MIN_LOOPS 1         /* measurement loop length for max msglength */
#define PRE_MSG_LOOPS 10    /* loop lenght for pre_msg */
#define TIMEPERLOOP 5e-1    /* = 500 msec */
#define MAX_FOR_LAST_TWO 60 /* seconds, limit for 2 bigest msg lengths */

#define MAX_RINGS     6
#define MAX_RANDOM   30
 
#define MIN_DIMS      2          /* 2 and 3 dimensional cartesian ... */
#define MAX_DIMS      3          /* ... patterns, not used in b_eff,  */
#define MAX_ACYCLIC  (MAX_DIMS)  /* ... but as additional information.*/
#define MAX_CYCLIC   (MAX_DIMS*(MAX_DIMS+3)/2-1)

#ifdef SHORT
#  undef  REPETITIONS
#  define REPETITIONS 2
#  undef  TIMEPERLOOP
#  define TIMEPERLOOP 5e-2
#  undef  MAX_RANDOM
#  define MAX_RANDOM 6
#endif /*SHORT*/

#ifdef TEST
#  undef  TIMEPERLOOP
#  define TIMEPERLOOP 5e-3
#  undef  MAX_FOR_LAST_TWO
#  define MAX_FOR_LAST_TWO 1
#  undef  MAX_RANDOM
#  define MAX_RANDOM 2
#endif /*TEST*/
 
#ifdef SHORT
# ifdef TEST
#  define TSTR "SHORT&TEST" 
# else
#  define TSTR "SHORT" 
# endif 
#else
# ifdef TEST
#  define TSTR "TEST" 
# else
#  define TSTR "" 
# endif 
#endif 
 
#ifndef PRINTLEVEL
#  define PRINTLEVEL 0
#endif /*PRINTLEVEL*/
 
#define PATTERNS     (MAX_RINGS+4+MAX_ACYCLIC+MAX_CYCLIC+MAX_RANDOM+1)
#define METHODS       3
#define MAXSR         6  /* maximum of calls to sendrecv in a pattern */
#define POSINF     1e30  /* a very high number to init minimum-calcul.*/

#define max(a,b) ( (a)>(b) ?   (a)  : (b) )
#define min(a,b) ( (a)<(b) ?   (a)  : (b) )
#define abs(a)   ( (a)< 0  ? (-(a)) : (a) )  

#define DEFAULT_PREFIX "b_eff"
#define DEFAULT_YRANGE "0.01:10000"


/* global variables: */ 
static int   first_ring_pat, first_random_pat; 

double logavg( double b_eff_pat[], int n) 
{
  double sum;  int i;
  sum = 0;
  for (i=0; i<n; i++) 
  { if (b_eff_pat[i] <= 0) 
    { printf("ERROR - trying to compute logavg( i=%d, val=%7.3f )\n",
             i, b_eff_pat[i]);
      return (999999999.0); 
    } 
    sum = sum + log( b_eff_pat[i] ); 
  } 
  return( exp( sum/n ) );
}

void logavg_pat( /*IN*/  double b_eff_pat[PATTERNS],
                 /*OUT*/ double *b_eff_ring, double *b_eff_random,
                         double *b_eff_final )
{
  double help[2];
  *b_eff_ring   = logavg(b_eff_pat+first_ring_pat,   MAX_RINGS);
  *b_eff_random = logavg(b_eff_pat+first_random_pat, MAX_RANDOM);
  help[0] = *b_eff_ring;  help[1] = *b_eff_random;
  *b_eff_final     = logavg(help, 2); 
} 

#ifdef _OPENMP
char *thread_support(int ival)
{  
   if (ival==MPI_THREAD_SINGLE) return("MPI_THREAD_SINGLE"); 
   else if (ival==MPI_THREAD_FUNNELED) return("MPI_THREAD_FUNNELED"); 
   else if (ival==MPI_THREAD_SERIALIZED) return("MPI_THREAD_SERIALIZED"); 
   else if (ival==MPI_THREAD_MULTIPLE) return("MPI_THREAD_MULTIPLE"); 
   else return("unknown value"); 
} 
#endif 
 
main(int argc, char **argv)
{
  /* methods: */ 
  int        max_methods, sndrcv_mthd, alltoall_mthd, nonblk_mthd; 
  char       *mthd_description[METHODS];
 
  /* patterns: */ 
  MPI_Comm   comm            [PATTERNS];
  int        sr_count        [PATTERNS];
  int        sr_sranks       [PATTERNS][MAXSR];
  int        sr_rranks       [PATTERNS][MAXSR];
  int        msg_count       [PATTERNS][MAXSR]; 
  int        do_participate  [PATTERNS]; /* 1=do, 0=not, 
                                           -1=not, but MPI_COMM_WORLD */
  int        participate_cnt [PATTERNS]; /* == number of nodes with
                                               (do_participate==1) */
  int        all_msg_cnt     [PATTERNS]; 
  char       *pat_description[PATTERNS];
  int        best_method     [REPETITIONS][PATTERNS]; 
  int        base_vals [MAXSR];
  int        *ranks;
/*int        best_bi_section_pat, worst_bi_section_pat;*/
  int        one_pingpong_pair_pat; 
/* global    first_ring_pat, first_random_pat; */
/*int        first_acyclic_pat,  first_cyclic_pat; */
/*int        acyclic_patterns,   cyclic_patterns;  */
  int        max_patterns;
   
  /* message lengths: */ 
  int        Lmax; 
  double     factor; 
  int        msglng        [MSGLENGTHS];
  int        looplngs      [MSGLENGTHS][REPETITIONS][METHODS];
  int        looplng, minlooplng;
  double     comm_times    [MSGLENGTHS][REPETITIONS][METHODS];
  double     min_looptimes [MSGLENGTHS][REPETITIONS][METHODS];
  double     comm_looptime;
  double     max_looptimes [MSGLENGTHS][REPETITIONS][METHODS];
 
  /* global send and receive buffer, cyclically used */ 
  int        buflng, base;
  char       *sendbuf, *recvbuf;

  /* organizing the measurements: */
  int        measure[REPETITIONS][PATTERNS][MSGLENGTHS][METHODS];
  int        repetitions[MSGLENGTHS]; 
  int        i_rep, i_msg, i_pat, i_loop, i_mthd, i_sr, i_ring, i_rq;
  int        pre_msg;
  int	     max_loops [2]; 
  /* helper for method 0 -- Sendrecv: */ 
  MPI_Status sts;
 
  /* helper for method 1 -- Alltoallv: */ 
  int        *scounts, *rcounts, *sdispls, *rdispls;
 
  /* helper for method 2 -- Irecv+Isend+Waitall: */ 
  MPI_Status  stss[2*MAXSR];
  MPI_Request rq[2*MAXSR];
 
  /* measurement results: */ 
  /* bw contains the results in each process,
     minbw is the minimum over all processes.
     Definition: minbw(L) = b(L) * sr_count / all_msg_cnt 
                 b(L) = ->see www.hlrs.de/mpi/b_eff        */
  double     bw    [REPETITIONS][PATTERNS][MSGLENGTHS][METHODS];
  double     minbw [REPETITIONS][PATTERNS][MSGLENGTHS][METHODS];
  double     avgbw [REPETITIONS][PATTERNS][MSGLENGTHS][METHODS];/*info*/
  double     maxbw [REPETITIONS][PATTERNS][MSGLENGTHS][METHODS];/*only*/

  /* analysis: */
  double     b_eff_pat_mthd[METHODS+1][PATTERNS];
  double     b_eff_rings_mthd[METHODS+1]; 
  double     b_eff_random_mthd[METHODS+1]; 
  double     b_eff_final_mthd[METHODS+1]; 
  double     b_eff_pat_rep[REPETITIONS+1][PATTERNS];
  double     b_eff_rings_rep[REPETITIONS+1]; 
  double     b_eff_random_rep[REPETITIONS+1]; 
  double     b_eff_final_rep[REPETITIONS+1]; 
  double     b_eff_pat_msg_mthd[MSGLENGTHS+1][METHODS][PATTERNS];
  double     b_eff_rings_msg_mthd[MSGLENGTHS+1][METHODS]; 
  double     b_eff_random_msg_mthd[MSGLENGTHS+1][METHODS]; 
  double     b_eff_final_msg_mthd[MSGLENGTHS+1][METHODS]; 
  double     b_eff_pat_msg[MSGLENGTHS+1][PATTERNS];
  double     b_eff_rings_msg[MSGLENGTHS+1]; 
  double     b_eff_random_msg[MSGLENGTHS+1]; 
  double     b_eff_final_msg[MSGLENGTHS+1]; 
  double     b_eff; 
  double     latency_onePingPong, b_asymp_onePingPong; 
 
  /* other variables: */ 
  int        myrank, size, comm_size, comm_myrank, rank;
  double     total_beg, total_end, expected_for_last_two; 
  double     time_beg, time_end, all_beg, all_end, all_delta;
  double     rep_sum, all_sum, rep_min, rep_min_sum;
  double     accum_bw, avgL_bw, maxmthd_bw, maxrep_bw; 
  struct utsname uname_info; 
  time_t     t_now; 
  int        i;             /*index variable for output (files[i]) - very bad :-(*/
  int        arg_error;
  char       *prefix;
  char       *fname;
  FILE       *fp; 
  FILE       *files[3];
  FILE       *plot, *gps, *tex;
  char       *plotfile;
  char       *yrange;

  double     ring_min, ring_max;
  double     random_min, random_max;
  char       *Label[4];

  /*Shortcuts: */
  int Msglng;    /* := msglng[i_msg] */
  int Msglng8;   /* := (Msglng + 7) / 8 * 8 */
 
#ifdef _OPENMP
  int required, provided, threads_per_mpi_process;
#endif 

  prefix = DEFAULT_PREFIX;
  yrange = DEFAULT_YRANGE;

  /*Headlines for the gnuplot script*/
  Label[0] = "Sndrcv";
  Label[1] = "Alltoal";
  Label[2] = "non-blk";
  Label[3] = "Best transfer method";

#ifdef _OPENMP
  required = MPI_THREAD_FUNNELED; 
  MPI_Init_thread(&argc, &argv, required, &provided);
#else 
  MPI_Init(&argc, &argv);
#endif 
  total_beg = MPI_Wtime();
  MPI_Comm_size(MPI_COMM_WORLD, &size);
  MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
  
  if (size == 1) 
  { 
    printf("ERROR - this benchmark needs at least two parallel"
           " MPI processes. You have started only %d process.\n",
           size);
    MPI_Abort(MPI_COMM_WORLD,1);
  }

  arg_error = 0;
  for (i=1; i<argc; i++)
  {
    if (strcmp(argv[i], "-yrange") == 0)
    {
      i++;
      if (i<argc) yrange=argv[i];
	  else arg_error++;
    }
    if (strcmp(argv[i], "-help") == 0)
    {
      if (myrank == 0) printf(HELP);
	  MPI_Finalize();
	  return(0);
    }
	if (strcmp(argv[i], "-f") == 0)
	{
	  i++;
      if (i<argc) prefix=argv[i];
	  else arg_error++;
	}
	else arg_error++;
  }
  
  if (arg_error > 0)
  {
    if (myrank == 0)
	{
	  printf("Invalid Option! Use 'b_eff -help' to get help.\n");
	}
	MPI_Finalize();
	return(1);
  }

  /*------------------------------------------------------------------*/
  /*       initialization of the threads (hybrid MPI+OpenMP only)     */
  /*------------------------------------------------------------------*/

#ifdef _OPENMP
#pragma omp parallel
 { /*begin of parallel block*/ 
#pragma omp master
  { /*begin of master thread block*/ 
    threads_per_mpi_process = omp_get_num_threads(); 
#endif 

  /*------------------------------------------------------------------*/
  /*               initialization of the protocol files               */
  /*------------------------------------------------------------------*/

  if (myrank == 0)
  {
    plotfile = prefix;
    strlen(prefix);
	fname = malloc(strlen(prefix)+7);
    strcpy(fname, prefix);
    strcat(fname, ".prot");      /*Protocol *.prot*/
    files[0] = fopen(fname,"w");
    strcpy(fname, prefix);
    strcat(fname, ".short");     /*Summary 7 Pages *.short*/
    files[1] = fopen(fname,"w");
    strcpy(fname, prefix);
    strcat(fname, ".sum");       /*Table *.sum*/
    files[2] = fopen(fname,"w");
    strcpy(fname, prefix);
    strcat(fname, ".plot");      /*GNUPLOT Data *.plot*/
    plot = fopen(fname,"w");
    strcpy(fname, prefix);
    strcat(fname, ".gps");       /*GNUPLOT Script *.gps*/
    gps = fopen(fname,"w");
    strcpy(fname, prefix);
    strcat(fname, ".tex");       /*LATEX  Source *.tex*/
    tex = fopen(fname,"w");
    free(fname);
    fp = files[0];
    
    for (i=0;i<=2;i++){
      fprintf(files[i],"\nSECTION-HEAD-BEGIN\n\n");
      fprintf(files[i],"b_eff.c, Revision %s from %s\n\n", Revision, Date);
      fprintf(files[i],"MEMORY_PER_PROCESSOR = %d MBytes  [1M = 1024*1024]\n",MEMORY_PER_PROCESSOR);
 
#ifdef _OPENMP
      fprintf(files[i],"\nHybrid MPI+OpenMP is used.\n");
      fprintf(files[i],"    MPI_Init_thread(required=%s): provided=%s",
                    thread_support(required), thread_support(provided));
        if (provided < required) 
          fprintf(files[i]," -- CAUTION, less than required"); 
        if (provided > required) 
          fprintf(files[i]," -- WARNING, more than required"); 
        fprintf(files[i],"\n");
      fprintf(files[i],"    Number of threads per MPI process = %d",
          threads_per_mpi_process);
        if (threads_per_mpi_process < 1)
          fprintf(files[i]," -- ERROR, less than 1 is invalid");
        if (threads_per_mpi_process == 1)
          fprintf(files[i]," -- WARNING, only one thread per MPI process!");
        if (threads_per_mpi_process > 1)
          fprintf(files[i]," (i.e., %d threads are sleeping while the master thread is running this benchmark)", threads_per_mpi_process-1); 
        fprintf(files[i],"\n\n");
#endif 
      fflush(files[i]); 
    }
  }

  /*------------------------------------------------------------------*/
  /*               initialization of the methods                      */
  /*------------------------------------------------------------------*/

# if (PRINTLEVEL >= 1) 
      printf("[%02d] initialization of the methods\n", myrank); fflush(stdout);
# endif 

  i_mthd = -1;
 
  sndrcv_mthd = -1;
  { /* Sendrecv */
    i_mthd++;
    sndrcv_mthd = i_mthd;
    mthd_description[i_mthd] = "Sndrcv";
  }
 
  alltoall_mthd = -1;  
# ifndef NOalltoallv
  { /* Alltoallv */
    i_mthd++;
    alltoall_mthd = i_mthd;
    mthd_description[i_mthd] = "Alltoal";
  }
# endif /*NOalltoallv*/ 
 
  nonblk_mthd = -1; 
# ifndef NOnonblk
  { /* Irecv+Isend+Waitall */
    i_mthd++;
    nonblk_mthd = i_mthd;
    mthd_description[i_mthd] = "non-blk";
  }
# endif /*NOnonblk*/ 

  max_methods = i_mthd+1;
  if ( max_methods > METHODS )
     { printf("ERROR - implementation error - METHODS should be =%d\n",
              max_methods);
       MPI_Abort(MPI_COMM_WORLD,1);
     }

  /*------------------------------------------------------------------*/
  /*               initialization of Lmax and buflng                  */
  /*------------------------------------------------------------------*/
 
  if ((sizeof(int) < 64) && (MEMORY_PER_PROCESSOR >= 128*128))
    /* restriction for base, buflng, sdispls, rdispls and MPI message 
       size: on systems with sizeof(int)==32 and MEMORY_PER_PROCESSOR
       larger than 16 GByte, the Lmax must be 128 MByte */ 
    Lmax = 128*1024*1024;
  else
    Lmax = MEMORY_PER_PROCESSOR * (1024*1024/128); 
  buflng = (Lmax+16L) * MAXSR;

  /*------------------------------------------------------------------*/
  /*              memory allocation and initialization                */
  /*------------------------------------------------------------------*/

# if (PRINTLEVEL >= 1) 
      printf("[%02d] memory allocation and initialization\n", myrank); fflush(stdout);
# endif 

  if ( !( sendbuf = (char *)malloc((size_t)buflng) )            ||
       !( recvbuf = (char *)malloc((size_t)buflng) )            ||
       !( scounts = (int *)malloc((size_t)(size*sizeof(int))) ) ||
       !( rcounts = (int *)malloc((size_t)(size*sizeof(int))) ) ||
       !( sdispls = (int *)malloc((size_t)(size*sizeof(int))) ) ||
       !( rdispls = (int *)malloc((size_t)(size*sizeof(int))) ) ||
       !( ranks   = (int *)malloc((size_t)(size*sizeof(int))) )   )
     { printf("ERROR - not enough space for buffers - "
              "is MEMORY_PER_PROCESSOR = %d MByte correct?\n",
              MEMORY_PER_PROCESSOR);
       MPI_Abort(MPI_COMM_WORLD,1);
     }
  if ( (MSGLENGTHS-1)*(MAXSR-1)+1 > 127 )
     { printf("ERROR - implementation error - check that maximum value "
              "for v2 (%d) does not exceed range of char (..127)\n",
              (MSGLENGTHS-1)*(MAXSR-1)+1);
       MPI_Abort(MPI_COMM_WORLD,1);
     }

  for (rank=0; rank<size; rank++)
  { scounts[rank]=0; rcounts[rank]=0; sdispls[rank]=0; rdispls[rank]=0;}

  for (i_msg=0; i_msg<MSGLENGTHS; i_msg++)
  { 
    repetitions[i_msg] = REPETITIONS; 
    for (i_rep=0; i_rep<REPETITIONS; i_rep++)
    { 
      for (i_mthd=0; i_mthd<max_methods; i_mthd++) 
      { 
        looplngs[i_msg][i_rep][i_mthd]      = 0; 
        comm_times[i_msg][i_rep][i_mthd]    = 0;
        min_looptimes[i_msg][i_rep][i_mthd] = POSINF;
        max_looptimes[i_msg][i_rep][i_mthd] = 0;
        for (i_pat=0; i_pat<PATTERNS; i_pat++) 
          measure[i_rep][i_pat][i_msg][i_mthd] = 1;
      } 
    } 
  } 
  for (i_rep=0; i_rep<REPETITIONS; i_rep++)
    for (i_pat=0; i_pat<PATTERNS; i_pat++) 
      best_method[i_rep][i_pat] = -1; /* i.e. all methods are used */

  /* to prohibit floating point exceptions in the unused entries 
     in the MPI_REDUCE operations */
    for (i_pat=0; i_pat<PATTERNS; i_pat++)
      for (i_rep=0; i_rep<REPETITIONS; i_rep++)
        for (i_msg=0; i_msg<MSGLENGTHS; i_msg++)
          for (i_mthd=0; i_mthd<METHODS; i_mthd++)
            bw[i_rep][i_pat][i_msg][i_mthd] = 0;
 
  latency_onePingPong = 0;  b_asymp_onePingPong = 0; 

  /*------------------------------------------------------------------*/
  /*          initialization of all communication patterns            */
  /*------------------------------------------------------------------*/

# if (PRINTLEVEL >= 1) 
      printf("[%02d] initialization of all communication patterns\n", myrank); fflush(stdout);
# endif 

# if (PRINTLEVEL >= 2) 
      printf("[%02d] - ring patterns\n", myrank); fflush(stdout);
# endif 

  i_pat = -1;
  { 
    MPI_Comm comm_1d;
    int medium_ring_size[MAX_RINGS];
    int number_of_rings;
    int low_ring_size, high_ring_size, low_count, high_count;
    int first_ring_size, second_ring_size, first_count, second_count;
    int period, prev, next, ring_size, base_rank;
 
    medium_ring_size[0] =       2; 
    medium_ring_size[1] =       4; 
    medium_ring_size[2] =       8; 
    medium_ring_size[3] = max( 16, size/4); 
    medium_ring_size[4] = max( 32, size/2); 
    medium_ring_size[5] =          size; 
 
    period = 1;
    MPI_Cart_create(MPI_COMM_WORLD, 1, &size, &period, 1, &comm_1d);
    MPI_Comm_rank(comm_1d, &comm_myrank);
 
    first_ring_pat = i_pat+1; 
    for (i_ring=0; i_ring<MAX_RINGS; i_ring++)
    {
      i_pat++;
      medium_ring_size[i_ring] = min(medium_ring_size[i_ring], size); 
      number_of_rings 
       =max(1,min((int)(1.0*size/medium_ring_size[i_ring]+0.4),size/2));
      low_ring_size = size/number_of_rings;
      high_ring_size = low_ring_size + 1;
      high_count = size - low_ring_size*number_of_rings; 
      low_count = number_of_rings - high_count;
      if ((low_ring_size%2) && (high_count>0))
       {first_ring_size=high_ring_size; second_ring_size=low_ring_size; 
        first_count=high_count;         second_count=low_count;}
      else 
       {first_ring_size=low_ring_size; second_ring_size=high_ring_size; 
        first_count=low_count;         second_count=high_count;}
      pat_description[i_pat] = (char *)malloc((size_t)20);
      sprintf(pat_description[i_pat],"ring-%1d*%1d%s",
         number_of_rings, first_ring_size,
         (second_count == 0 ? "fix" :
          (second_ring_size>first_ring_size ? "&+1" : "&-1")) ); 
      if (comm_myrank < (first_count*first_ring_size)) 
      { ring_size = first_ring_size; 
        base_rank = comm_myrank/ring_size*ring_size;
      }else{
        ring_size = second_ring_size;
        base_rank = (comm_myrank-first_count*first_ring_size)
                    /ring_size*ring_size + first_count*first_ring_size;
      }
      next = (comm_myrank-base_rank+1) % ring_size + base_rank;
      prev = (comm_myrank-base_rank+ring_size-1)%ring_size + base_rank;
      comm[i_pat]           = comm_1d;
      sr_count[i_pat]       = 2; /*Ping-Pong*/
      sr_sranks[i_pat][0]   = next; /*Ping*/
      sr_rranks[i_pat][0]   = prev;
      msg_count[i_pat][0]   = size;
      sr_sranks[i_pat][1]   = prev; /*Pong*/
      sr_rranks[i_pat][1]   = next;
      msg_count[i_pat][1]   = size;
      do_participate[i_pat] = 1;
      participate_cnt[i_pat]= size;
      all_msg_cnt[i_pat]    = size * 2;
    } 
  } 

# if (PRINTLEVEL >= 2) 
      printf("[%02d] - random patterns\n", myrank); fflush(stdout);
# endif 

  { /* random patterns with cyclic-1dim pattern */
    int i_rand, i, j;
    MPI_Group world_group, random_group;
    MPI_Comm random_comm;

    srand48((long)(time((time_t *) 0))); 
    first_random_pat = i_pat+1;
    for (i_rand=0; i_rand<MAX_RANDOM+1; i_rand++)
                                 /* ^^- worst distribution, not random*/
    {
      if (myrank==0)
        if (i_rand<MAX_RANDOM) 
        { /* random pattern */
          for (i=0; i<size; i++) ranks[i] = -1;
          for (i=0; i<size; i++)
          {
            j = (int)(drand48() * size);
            while (ranks[j] != -1) j = (j+1) % size;
            ranks[j] = i;
          }
        }
        else
        { /* Worst case distribution */
          /* THIS PATTERN IS NOT USED IN THE COMPUTATION OF b_eff ! */
          for (i=0; i<size; i++) 
            ranks[i] = ( i%2 == 0 ? i/2 : size-1-i/2 ); 
        } 
      MPI_Bcast(ranks, size, MPI_INT, 0, MPI_COMM_WORLD);
      MPI_Comm_group(MPI_COMM_WORLD, &world_group);
      MPI_Group_incl(world_group, size, ranks, &random_group);
      MPI_Comm_create(MPI_COMM_WORLD, random_group, &random_comm);
      MPI_Group_free(&random_group); 
      MPI_Comm_rank(random_comm, &comm_myrank);
      i_pat++;
      comm[i_pat]           = random_comm;
      sr_count[i_pat]       = 2; /*Ping-Pong*/
      sr_sranks[i_pat][0]   = (comm_myrank +1) % size;       /*Ping*/
      sr_rranks[i_pat][0]   = (comm_myrank -1 +size) % size;
      sr_sranks[i_pat][1]   = (comm_myrank -1 +size) % size; /*Pong*/
      sr_rranks[i_pat][1]   = (comm_myrank +1) % size;
      msg_count[i_pat][0]   = size;
      msg_count[i_pat][1]   = size;
      do_participate[i_pat] = 1;
      participate_cnt[i_pat]= size;
      all_msg_cnt[i_pat]    = size * 2; 
      pat_description[i_pat]= "random-cyc-1dim";
      if (i_rand==MAX_RANDOM) pat_description[i_pat] = "worst-cyc-1dim"; 
    }
  }
# ifdef NOworstcyc
   i_pat--; /*remove last pattern = worst-cyc-1dim pattern */
# endif 

# if (PRINTLEVEL >= 2) 
      printf("[%02d] - other patterns\n", myrank); fflush(stdout);
# endif 

  {
    int period;
    MPI_Comm comm_1d;
    period = 1;
    MPI_Cart_create(MPI_COMM_WORLD, 1, &size, &period, 1, &comm_1d);
    MPI_Comm_rank(comm_1d, &comm_myrank);

    if (myrank==0)
    {
      fprintf(files[0],"1-dim-paterns: size = %d\n",size);
      fprintf(files[0],"1-dim-paterns: size = %d\n",size);
    }

#  ifdef FURTHERPATTERNS 
    /* 1-dim., cyclic, each node makes a ping-pong with its right neighbor */
    /* This pattern is not used in the computation of b_eff !  */
    /* But its result should be the same of "cyclic-1dim-all". */
    i_pat++;
    comm[i_pat]           = comm_1d;
    sr_count[i_pat]       = 2; /*Ping-Pong*/
    sr_sranks[i_pat][0]   = (comm_myrank + 1) % size;         /*Ping*/
    sr_rranks[i_pat][0]   = (comm_myrank - 1 + size) % size;
    msg_count[i_pat][0]   = size;
    sr_sranks[i_pat][1]   = (comm_myrank - 1 + size) % size;  /*Pong*/
    sr_rranks[i_pat][1]   = (comm_myrank + 1) % size;
    msg_count[i_pat][1]   = size;
    do_participate[i_pat] = 1;
    participate_cnt[i_pat]= size;
    all_msg_cnt[i_pat]    = size * 2;
    pat_description[i_pat]= "cyclic-1dim-x";
#  endif /*FURTHERPATTERNS*/ 

#  ifndef NObestBIsection 
    /* 1-dim., best bi-section: each second node makes a ping-pong
                               with its right neighbor */
    i_pat++;  /* best_bi_section_pat = i_pat; */
    comm_size = size/2*2;
    comm[i_pat] = comm_1d;
    sr_count[i_pat] = 2; /*Ping-Pong*/
    do_participate[i_pat] = 1;
    participate_cnt[i_pat] = comm_size;
    all_msg_cnt[i_pat] = comm_size;
    pat_description[i_pat] = "best  bi-section";
    if (comm_myrank >= comm_size)
    { sr_count[i_pat] = 0; /* with odd size, last node cannot be used */
      do_participate[i_pat] = -1; 
    }
    else 
    { if ((comm_myrank % 2) == 0)
      { /* ping=send - pong=recv */
        sr_sranks[i_pat][0] = comm_myrank + 1;
        sr_rranks[i_pat][0] = MPI_PROC_NULL;
        sr_sranks[i_pat][1] = MPI_PROC_NULL;
        sr_rranks[i_pat][1] = comm_myrank + 1;
      }
      else
      { /* ping=recv - pong=send */
        sr_sranks[i_pat][0] = MPI_PROC_NULL;
        sr_rranks[i_pat][0] = comm_myrank - 1;
        sr_sranks[i_pat][1] = comm_myrank - 1;
        sr_rranks[i_pat][1] = MPI_PROC_NULL;
      }
      msg_count[i_pat][0]   = comm_size / 2;
      msg_count[i_pat][1]   = comm_size / 2;
    }
#  endif /*NObestBIsection*/ 

#  ifndef NOworstBIsection 
    /* 1-dim., worst bi-section: first node makes ping-pong with last,
                                second with last but one, ... */
    /* THIS PATTERN IS NOT USED IN THE COMPUTATION OF b_eff ! */
    i_pat++;  /* worst_bi_section_pat = i_pat; */
    comm_size = size/2*2;
    comm[i_pat] = comm_1d;
    sr_count[i_pat] = 2; /*Ping-Pong*/
    do_participate[i_pat] = 1;
    participate_cnt[i_pat] = comm_size;
    all_msg_cnt[i_pat] = comm_size;
    pat_description[i_pat] = "worst bi-section";
    if (comm_myrank >= comm_size)
    { sr_count[i_pat] = 0; /* with odd size, last node cannot be used */
      do_participate[i_pat] = -1; 
    }
    else
    { if (comm_myrank < (comm_size/2))
      { /* ping=send - pong=recv */
        sr_sranks[i_pat][0] = comm_size - comm_myrank - 1;
        sr_rranks[i_pat][0] = MPI_PROC_NULL;
        sr_sranks[i_pat][1] = MPI_PROC_NULL;
        sr_rranks[i_pat][1] = comm_size - comm_myrank - 1;
      }
      else
      { /* ping=recv - pong=send */
        sr_sranks[i_pat][0] = MPI_PROC_NULL;
        sr_rranks[i_pat][0] = comm_size - comm_myrank - 1;
        sr_sranks[i_pat][1] = comm_size - comm_myrank - 1;
        sr_rranks[i_pat][1] = MPI_PROC_NULL;
      }
      msg_count[i_pat][0]   = comm_size / 2;
      msg_count[i_pat][1]   = comm_size / 2;
    }
#  endif /*NOworstBIsection*/ 

#  ifndef NOonePingPongPair
    /* 1-dim., only node 0 and 1 execute a ping-pong */
    i_pat++;  one_pingpong_pair_pat = i_pat;
    comm_size = 2;
    comm[i_pat] = comm_1d;
    sr_count[i_pat] = 2; /*Ping-Pong*/
    do_participate[i_pat] = 1;
    participate_cnt[i_pat] = comm_size;
    all_msg_cnt[i_pat] = comm_size;
    pat_description[i_pat] = "one PingPong Pair";
    if (comm_myrank >= comm_size)
    { sr_count[i_pat] = 0; /* with odd size, last node cannot be used */
      do_participate[i_pat] = -1; 
    }
    else 
    { if ((comm_myrank % 2) == 0)
      { /* ping=send - pong=recv */
        sr_sranks[i_pat][0] = comm_myrank + 1;
        sr_rranks[i_pat][0] = MPI_PROC_NULL;
        sr_sranks[i_pat][1] = MPI_PROC_NULL;
        sr_rranks[i_pat][1] = comm_myrank + 1;
      }
      else
      { /* ping=recv - pong=send */
        sr_sranks[i_pat][0] = MPI_PROC_NULL;
        sr_rranks[i_pat][0] = comm_myrank - 1;
        sr_sranks[i_pat][1] = comm_myrank - 1;
        sr_rranks[i_pat][1] = MPI_PROC_NULL;
      }
      msg_count[i_pat][0]   = comm_size / 2;
      msg_count[i_pat][1]   = comm_size / 2;
    }
    for (i_rep=0; i_rep<REPETITIONS; i_rep++)
      for (i_msg=0; i_msg<MSGLENGTHS; i_msg++)
         for (i_mthd=0; i_mthd<max_methods; i_mthd++) 
           if (i_mthd != sndrcv_mthd) /* only Sendrecv ! */
             measure[i_rep][i_pat][i_msg][i_mthd] = 0;
#  else
    one_pingpong_pair_pat = -1; 
#  endif /*NOonePingPongPair*/ 
  }

  {/* dims 2-3, acyclic/cyclic, each positive direction
      and all positive directions:
      ping-pong with each neighbor in the direction */
    int ndims, period, direction, src, dest, i, j;
    int dims[MAX_DIMS], periods[MAX_DIMS];
    MPI_Comm comm_cart;

    /*acyclic_patterns = 0; cyclic_patterns = 0;*/
    for (period=0; period<2; period++)
    {
#     ifdef NOacyclic
        if (period==0) continue;
#     endif /*NOacyclic*/ 
#     ifdef NOcyclic
        if (period==1) continue;
#     endif /*NOcyclic*/ 
 
      /*if (period==0) first_acyclic_pat = i_pat+1;
        else           first_cyclic_pat  = i_pat+1; */
      for (ndims = MIN_DIMS; ndims <= MAX_DIMS; ndims++)
      {
        comm_size = size + 1;
        do
        { comm_size--;
          for (i=0; i<ndims; i++) dims[i] = 0;
          MPI_Dims_create(comm_size, ndims, dims);
        } while( ! /*until:*/ ( (dims[ndims-1] > 1)
                                || ((comm_size<8) && ((comm_size%2)==0))
                                || (comm_size<4)
                              ) ); 
        for (i=0; i<ndims; i++) periods[i] = period;
        MPI_Cart_create(MPI_COMM_WORLD,ndims,dims,periods,1,&comm_cart);
        if (myrank==0 && period==0)
        { 
	  for (i=0; i<=2; i++){
	    fprintf(files[i],"%1d-dim-patterns: size = %1d",ndims,dims[0]);
	    for (j=1; j<ndims; j++) fprintf(files[i]," * %1d",dims[j]);
	    fprintf(files[i],"\n");
	  }
        }

        if ((ndims>1) && (period==1))
        { 
          for (direction=0; direction<ndims; direction++)
          {
           if (dims[direction] > 1)
           {
            i_pat++;
            /*if(period==0)acyclic_patterns++; else cyclic_patterns++;*/
            pat_description[i_pat] = (char *)malloc((size_t)20);
            sprintf(pat_description[i_pat],"%s-%1ddim-%c",
                     (period ? "cyclic" : "acyclic"), ndims,
                     (char)(direction+(int)'x'));
            participate_cnt[i_pat]= comm_size;
            if (comm_cart == MPI_COMM_NULL)
              { sr_count[i_pat] = 0; do_participate[i_pat] = 0; }
            else
            {
              MPI_Cart_shift(comm_cart, direction, 1, &src, &dest);
              comm[i_pat]           = comm_cart;
              sr_count[i_pat]       = 2; /*Ping-Pong*/
              sr_sranks[i_pat][0]   = dest; /*Ping*/
              sr_rranks[i_pat][0]   = src;
              sr_sranks[i_pat][1]   = src;  /*Pong*/
              sr_rranks[i_pat][1]   = dest;
              do_participate[i_pat] = 1;
              msg_count[i_pat][0]   = (dims[direction]-1+periods[direction])
                                      * comm_size / dims[direction];
              msg_count[i_pat][1]   = msg_count[i_pat][0];
              all_msg_cnt[i_pat] = msg_count[i_pat][0]+msg_count[i_pat][1];
            }
           }
          }
        } 

        i_pat++;
        /*if (period==0) acyclic_patterns++; else cyclic_patterns++;*/
        pat_description[i_pat] = (char *)malloc((size_t)20);
        sprintf(pat_description[i_pat],"%s-%1ddim-all",
                 (period ? "cyclic" : "acyclic"), ndims);
        participate_cnt[i_pat]= comm_size; 
        if (comm_cart == MPI_COMM_NULL)
          { sr_count[i_pat] = 0; do_participate[i_pat] = 0; }
        else
        {
          do_participate[i_pat] = 1;
          all_msg_cnt[i_pat]    = 0; /*init sum*/
          comm[i_pat]           = comm_cart;
          sr_count[i_pat]       = 0;
          i_sr = 0;
          for (direction=0; direction<ndims; direction++)
           if (dims[direction] >= 2) 
           {
            MPI_Cart_shift(comm_cart, direction, 1, &src, &dest);
            sr_count[i_pat]          = sr_count[i_pat] + 2;/*Ping-Pong*/
            sr_sranks[i_pat][i_sr]   = dest; /*Ping*/
            sr_rranks[i_pat][i_sr]   = src;
            sr_sranks[i_pat][i_sr+1] = src;  /*Pong*/
            sr_rranks[i_pat][i_sr+1] = dest;
            msg_count[i_pat][i_sr] =(dims[direction]-1+periods[direction])
                                    * comm_size / dims[direction];
            msg_count[i_pat][i_sr+1] = msg_count[i_pat][i_sr]; 
            all_msg_cnt[i_pat] = all_msg_cnt[i_pat] +
                      msg_count[i_pat][i_sr] + msg_count[i_pat][i_sr+1];
            i_sr = i_sr+2; 
           }
        }
      }
    }
  }

  max_patterns = i_pat+1;
  if ( max_patterns > PATTERNS )
     { printf("ERROR - implementation error - PATTERNS should be =%d\n",
              max_patterns);
       MPI_Abort(MPI_COMM_WORLD,1);
     }

# if (PRINTLEVEL >= 2) 
      printf("[%02d] - Removing pong\n", myrank); fflush(stdout);
# endif 

  { /* Removing "pong" send-receive-pairs in a pattern, if the
       pong has the same destination as the ping, because in this
       case the collective implementation (Alltoallv) of the
       communication measurement cannot send ping and pong,
       because one call to Alltoallv can transfer only one message
       between two nodes.
       This case is only valid, if the number of nodes in this
       cartesian direction is two and this direction is cyclic.
       The measurement algorithm requires that ping and pong are stored
       contigues in sr_sranks and sr_rranks. */
    int i;
    for (i_pat=0; i_pat<max_patterns; i_pat++)
    {
      for (i_sr=sr_count[i_pat]-2 /*last ping*/; i_sr>=0; i_sr=i_sr-2)
      {
        if (sr_sranks[i_pat][i_sr] == sr_sranks[i_pat][i_sr+1])
        { /* then remove the current pong: */
          all_msg_cnt[i_pat] = all_msg_cnt[i_pat] -
                               msg_count[i_pat][i_sr];
          for (i=i_sr+2; i<sr_count[i_pat]; i++)
          { /* move higher ping-pong pairs down to overwrite the current pong:*/
            sr_sranks[i_pat][i-1] = sr_sranks[i_pat][i];
            sr_rranks[i_pat][i-1] = sr_rranks[i_pat][i];
            msg_count[i_pat][i-1] = msg_count[i_pat][i];
          }
          sr_count[i_pat]--;
        }
      }
    }
  }

 
  /*------------------------------------------------------------------*/
  /*     initialization of message length and loop length values      */
  /*------------------------------------------------------------------*/

# if (PRINTLEVEL >= 1) 
      printf("[%02d] initialization of message length and loop length values\n", myrank); fflush(stdout);
# endif 

  { double lng;
 
    /* i_msg=0 (L=2**0)  until  i_msg=MAX_FACT_2 (L=2**MAX_FACT_2) */
    msglng[0] = 1 /* 2**0 */;
    for (i_msg=1; i_msg<=MAX_FACT_2; i_msg++)
      msglng[i_msg] = 2 * msglng[i_msg-1];

    /* i_msg=MAX_FACT_2+1 (L=2**MAX_FACT_2 * factor**1) until
       i_msg=MSGLENGTHS-1 (L=2**MAX_FACT_2 * factor**(MSGLENGTHS-1-MAX_FACT_2) = Lmax) */
    lng = msglng[MAX_FACT_2];
    factor = pow( ((double)(Lmax)) / lng,
                  1.0/(MSGLENGTHS-1-MAX_FACT_2) );
    for (i_msg=MAX_FACT_2+1; i_msg<MSGLENGTHS; i_msg++)
      { lng = lng * factor;  msglng[i_msg] = (int)(lng+0.5); }
  }

  if (myrank == 0)
  { 
    for (i=0; i<=2; i++){
      fprintf(files[i],"Used message lengths:\n");
      fprintf(files[i],"msglng=");
      for (i_msg=0; i_msg<MSGLENGTHS; i_msg++)
        fprintf(files[i]," %1d",msglng[i_msg]);
      fprintf(files[i],"\n");
      fprintf(files[i],"Used methods:");
      for (i_mthd=0; i_mthd<max_methods; i_mthd++) 
        fprintf(files[i],"  %1d=%s",i_mthd,mthd_description[i_mthd]); 
      fprintf(files[i],"\n");
      fprintf(files[i],"Used patterns:");
      for (i_pat=0; i_pat<max_patterns; i_pat++)
	  {   
	    if ((i_pat%4 == 0) && (i_pat>0)) fprintf(files[i],"\n              ");
        fprintf(files[i]," %2d=%-17s",i_pat,pat_description[i_pat]); 
      } 
      fprintf(files[i],"\n\n");
	  fprintf(files[i],"               0-5   used for ring pattern average of b_eff\n");
	  fprintf(files[i],"               6-35  used for random pattern average of b_eff\n");
	  fprintf(files[i],"               36-47 only reported, not used for b_eff average\n");
      fprintf(files[i],"\nSECTION-HEAD-END\n\n");
    }
  }
 

 
  /*------------------------------------------------------------------*/
  /*                         measurement loop                         */
  /*------------------------------------------------------------------*/

# if (PRINTLEVEL >= 1) 
      printf("[%02d] measurement loop\n", myrank); fflush(stdout);
# endif 

  if (myrank == 0) fprintf(fp,"\nSECTION-LOOP-BEGIN\n\n");
  MPI_Barrier(MPI_COMM_WORLD);
  all_beg = MPI_Wtime();
  base = 0;
  for (i_rep=0; i_rep<REPETITIONS; i_rep++)
  {
#   if (PRINTLEVEL >= 1) 
      printf("[%02d]%4.0fs i_rep=%2d           -- next i_rep\n",
             myrank, MPI_Wtime()-all_beg,i_rep); fflush(stdout);
#   endif 
 
    pre_msg = 1;	/* first iteration only for test! (looplng) */
	for (i_msg=0; i_msg<MSGLENGTHS; i_msg++)
    {
#    if (PRINTLEVEL >= 2) 
       printf("[%02d]%4.0fs i_rep=%2d i_msg=%2d      -- next i_msg\n",
              myrank, MPI_Wtime()-all_beg,i_rep, i_msg); fflush(stdout);
#    endif 
     if (i_rep < repetitions[i_msg]) 
     { 
      Msglng = msglng[i_msg];
      Msglng8 = (Msglng + 7) / 8 * 8;
      if (myrank==0) 
      { 
        for (i_mthd=0; i_mthd<max_methods; i_mthd++) 
        {
          if (i_msg==0)
		    if (pre_msg==1) looplng = PRE_MSG_LOOPS;
			else
	        {
			  max_loops[i_mthd] = PRE_MSG_LOOPS * (TIMEPERLOOP/comm_times[i_msg][i_rep][i_mthd]);
			  if (max_loops[i_mthd] < PRE_MSG_LOOPS) max_loops[i_mthd] = PRE_MSG_LOOPS;
		      looplng = max_loops[i_mthd]/2;
			}
          else 
          { 
            looplng = looplngs[i_msg-1][i_rep][i_mthd]; 
            comm_looptime = comm_times[i_msg-1][i_rep][i_mthd];
            if (comm_looptime > 0)
               looplng = max(MIN_LOOPS,
                             min(looplng 
                                   / (i_msg <= MAX_FACT_2 ? 2.0 : factor)
                                   * max(1.,TIMEPERLOOP/comm_looptime),
                                 max_loops[i_mthd]));
          } 
#ifndef NOshortlasttwo
          if (i_msg == (MSGLENGTHS-2))
          { 
            /* expected time for the last two i_msg:*/
            expected_for_last_two = 
               min_looptimes[i_msg-1][i_rep][i_mthd] * max_patterns * 4./*worse patterns*/
                * (factor/*last_but_one*/ + factor*factor/*last i_msg*/)
                * REPETITIONS * max_methods 
                / looplngs[i_msg-1][i_rep][i_mthd];
            if (expected_for_last_two >  MAX_FOR_LAST_TWO) looplng = 0;
#           if (PRINTLEVEL >= 2) 
             if (looplng==0) printf("expected time=%9.3e for "
               "i_rep=%1d, i_msg=%2d, i_mthd=%1d\n",
               expected_for_last_two, i_rep, i_msg, i_mthd);
#           endif 
          }
#endif
	  /* printf("looplng i_msg=%i i_rep=%i i_mthd=%i: %i \n", i_msg, i_rep, i_mthd, looplng); */
          looplngs[i_msg][i_rep][i_mthd] = looplng; 
        } /* endfor i_mthd */
      } /* endif (myrank==0) */ 
 
      MPI_Bcast(looplngs[i_msg][i_rep], max_methods, MPI_INT, 0, MPI_COMM_WORLD); 
 
      minlooplng = PRE_MSG_LOOPS;
      for (i_mthd=0; i_mthd<max_methods; i_mthd++) 
        minlooplng = min( minlooplng, looplngs[i_msg][i_rep][i_mthd]);
      if (minlooplng==0)
      {
		/* the last two i_msg iterations will be shortened:
           i_msg == MSGLENGTHS-2 and -1: 
             only the best method of the last iteration (i_msg==...-3)
             is used, and looplng=1;
           i_msg == MSGLENGTHS-1:
             only MIN_REPETITIONS are used.
        */ 
        if (myrank==0) 
        { /* choosing best method for each pattern:*/ 
          for (i_pat=0; i_pat<max_patterns; i_pat++)
          { best_method[i_rep][i_pat] = 0;
            for (i_mthd=1; i_mthd<max_methods; i_mthd++)
             if (measure[i_rep][i_pat][i_msg-1][i_mthd]) 
              if (bw[i_rep][i_pat][i_msg-1][i_mthd] >=
                  bw[i_rep][i_pat][i_msg-1][best_method[i_rep][i_pat]]) 
                best_method[i_rep][i_pat] = i_mthd; 
          } 
        } 
        MPI_Bcast(best_method[i_rep], max_patterns,MPI_INT, 0, MPI_COMM_WORLD);
        repetitions[i_msg+1] = MIN_REPETITIONS; 
        for (i_mthd=0; i_mthd<max_methods; i_mthd++)
        { 
          looplngs[i_msg][i_rep][i_mthd] = 1;
          looplngs[i_msg+1][i_rep][i_mthd] = 
                                 (i_rep < repetitions[i_msg+1] ? 1 : 0);
          for (i_pat=0; i_pat<max_patterns; i_pat++)
          {
            measure[i_rep][i_pat][i_msg][i_mthd] = 
                                 (best_method[i_rep][i_pat]==i_mthd); 
            measure[i_rep][i_pat][i_msg+1][i_mthd] = 
                                 (best_method[i_rep][i_pat]==i_mthd)
                                 && (i_rep < repetitions[i_msg+1]); 
          } 
        } 
      } /* endif (minlooplng==0) */
 
      for (i_pat=0; i_pat<max_patterns; i_pat++)
      {
#      if (PRINTLEVEL >= 3) 
         printf("[%02d]%4.0fs i_rep=%2d i_msg=%2d i_pat=%2d -- next i_pat\n",
                myrank, MPI_Wtime()-all_beg,i_rep, i_msg, i_pat); fflush(stdout);
#      endif 
 
       for (i_mthd=0; i_mthd<max_methods; i_mthd++) 
       if (measure[i_rep][i_pat][i_msg][i_mthd])
       { 
        MPI_Barrier(MPI_COMM_WORLD);
#       if (PRINTLEVEL >= 4) 
         printf("[%02d]%4.0fs i_rep=%2d i_msg=%2d i_pat=%2d -- mthd %1d begin\n"
                ,myrank, MPI_Wtime()-all_beg, i_rep, i_msg, i_pat, i_mthd); fflush(stdout);
#       endif 
 
        if (i_mthd == sndrcv_mthd)
        { /* method 0: sendrecv */
         if (do_participate[i_pat] == 1)
         { 
          looplng = looplngs[i_msg][i_rep][i_mthd]; 
          for (i_loop = (i_msg < WITH_PRELOOPS ? -PRE_LOOPS : 0); 
               i_loop<looplng; i_loop++)
          {
           if (i_loop == 0) time_beg = MPI_Wtime();
           for (i_sr=0; i_sr<sr_count[i_pat]; i_sr++)
           { register char v1=i_sr*i_msg+1, v2=2*v1-1;
                      /* both values must be identical for Msglng==1 */
             register int dest=sr_sranks[i_pat][i_sr];
             register int src =sr_rranks[i_pat][i_sr];
             if ((base+Msglng8) > buflng) base = 0;
             sendbuf[base] = v1; sendbuf[base+Msglng-1] = v2;
#            if (PRINTLEVEL >= 5) 
               printf("[%02d]%4.0fs i_rep=%2d i_msg=%2d i_pat=%2d -- mthd 0 "
                      "loop=%3d sr=%1d before SendRecv\n", myrank, MPI_Wtime()-all_beg,i_rep,
                      i_msg, i_pat, i_loop, i_sr); fflush(stdout);
#            endif 
             if (src == MPI_PROC_NULL)
               MPI_Send(sendbuf+base,Msglng,MPI_BYTE,dest,i_sr,comm[i_pat]);
             else if (dest == MPI_PROC_NULL)
               MPI_Recv(recvbuf+base,Msglng,MPI_BYTE,src,i_sr,comm[i_pat],&sts);
             else
               MPI_Sendrecv(
                 sendbuf+base,Msglng,MPI_BYTE,dest,i_sr,
                 recvbuf+base,Msglng,MPI_BYTE,src,i_sr,comm[i_pat],&sts);
#            if (PRINTLEVEL >= 5) 
               printf("[%02d]%4.0fs i_rep=%2d i_msg=%2d i_pat=%2d -- mthd 0 "
                      "loop=%3d sr=%1d after  SendRecv\n", myrank, MPI_Wtime()-all_beg,i_rep,
                      i_msg, i_pat, i_loop, i_sr); fflush(stdout);
#            endif 
             if ((sr_rranks[i_pat][i_sr] != MPI_PROC_NULL) &&
                 ( (recvbuf[base] != v1) ||
                   (recvbuf[base+Msglng-1] != v2) ) )
                { printf("ERROR - invalid message content after "
                         "MPI_Sendrecv - myrank=%1d i_rep=%1d i_msg=%1d "
                         "i_pat=%1d i_sr=%1d i_loop=%1d Msglng=%1d "
                         "buf=(%1d %1d)!=(%1d %1d)\n",
                         myrank,i_rep,i_msg,i_pat,i_sr,i_loop,Msglng,
                         (int)recvbuf[base], (int)recvbuf[base+Msglng-1],
                         (int)v1, (int)v2);
                  MPI_Abort(MPI_COMM_WORLD,1);
                }
             base = base + Msglng8;
           } /* endfor i_sr */
          } /* endfor i_loop */
          time_end = MPI_Wtime();
         } /* endif do_participate[i_pat] == 1 */
        } /* endif i_mthd==sndrcv_mthd */

        if (i_mthd == alltoall_mthd)
        { /* method 1: alltoallv */
         for (i_sr=0; i_sr<sr_count[i_pat]; i_sr++)
         { if (sr_sranks[i_pat][i_sr] != MPI_PROC_NULL)
             scounts[sr_sranks[i_pat][i_sr]] = Msglng;
           if (sr_rranks[i_pat][i_sr] != MPI_PROC_NULL)
             rcounts[sr_rranks[i_pat][i_sr]] = Msglng;
         }
         if (do_participate[i_pat] != 0)
         { 
          looplng = looplngs[i_msg][i_rep][i_mthd]; 
          for (i_loop = (i_msg < WITH_PRELOOPS ? -PRE_LOOPS : 0); 
               i_loop<looplng; i_loop++)
          {
           if (i_loop == 0) time_beg = MPI_Wtime();
           for (i_sr=0; i_sr<sr_count[i_pat]; i_sr++)
           {register char v1=i_sr*i_msg+1, v2=2*v1-1;
                      /* both values must be identical for Msglng==1 */
             if ((base+Msglng8) > buflng) base = 0;
             sendbuf[base] = v1; sendbuf[base+Msglng-1] = v2;
             base_vals[i_sr] = base;
             if (sr_sranks[i_pat][i_sr] != MPI_PROC_NULL) 
               sdispls[sr_sranks[i_pat][i_sr]] = base;
             if (sr_rranks[i_pat][i_sr] != MPI_PROC_NULL) 
               rdispls[sr_rranks[i_pat][i_sr]] = base;
             base = base + Msglng8;
           } /* endfor i_sr */
#          if (PRINTLEVEL >= 6) 
            for (rank=0; rank<size; rank++)
              printf("[%02d]%4.0fs neighbor=%1d, sc=%1d sd=%1d rc=%1d rd=%1d\n",
                     myrank, MPI_Wtime()-all_beg, rank, scounts[rank], sdispls[rank],
                                   rcounts[rank], rdispls[rank]);
#          endif 
#          if (PRINTLEVEL >= 5) 
             printf("[%02d]%4.0fs i_rep=%2d i_msg=%2d i_pat=%2d -- mthd 1 "
                    "loop=%3d      before Alltoallv\n", myrank, MPI_Wtime()-all_beg,i_rep, 
                    i_msg, i_pat, i_loop); fflush(stdout);
#          endif 
           MPI_Alltoallv(sendbuf,scounts,sdispls,MPI_BYTE,
                         recvbuf,rcounts,rdispls,MPI_BYTE,comm[i_pat]);
#          if (PRINTLEVEL >= 5) 
             printf("[%02d]%4.0fs i_rep=%2d i_msg=%2d i_pat=%2d -- mthd 1 "
                    "loop=%3d      after  Alltoallv\n", myrank, MPI_Wtime()-all_beg,i_rep, 
                    i_msg, i_pat, i_loop); fflush(stdout);
#          endif 
           for (i_sr=0; i_sr<sr_count[i_pat]; i_sr++)
           { if ((sr_rranks[i_pat][i_sr] != MPI_PROC_NULL) &&
                 ( (recvbuf[base_vals[i_sr]]
                     != sendbuf[base_vals[i_sr]]) ||
                   (recvbuf[base_vals[i_sr]+Msglng-1]
                     != sendbuf[base_vals[i_sr]+Msglng-1]) ) )
                { printf("ERROR - invalid message content after "
                         "MPI_Alltoallv - myrank=%1d i_rep=%1d i_msg=%1d"
                         " i_pat=%1d i_sr=%1d i_loop=%1d Msglng=%1d "
                         "buf=(%1d %1d)!=(%1d %1d)\n",
                         myrank,i_rep,i_msg,i_pat,i_sr,i_loop,Msglng,
                         (int)recvbuf[base_vals[i_sr]],
                         (int)recvbuf[base_vals[i_sr]+Msglng-1],
                         (int)sendbuf[base_vals[i_sr]],
                         (int)sendbuf[base_vals[i_sr]+Msglng-1]);
                  MPI_Abort(MPI_COMM_WORLD,1);
                }
           } /* endfor i_sr */
          } /* endfor i_loop */
          time_end = MPI_Wtime();
          for (i_sr=0; i_sr<sr_count[i_pat]; i_sr++)
          { 
            if (sr_sranks[i_pat][i_sr] != MPI_PROC_NULL) 
              { scounts[sr_sranks[i_pat][i_sr]] = 0;
                sdispls[sr_sranks[i_pat][i_sr]] = 0; }
            if (sr_rranks[i_pat][i_sr] != MPI_PROC_NULL) 
              { rcounts[sr_rranks[i_pat][i_sr]] = 0;
                rdispls[sr_rranks[i_pat][i_sr]] = 0; }
          }
         } /* endif do_participate[i_pat] != 0 */
        } /* endif i_mthd==alltoall_mthd */

        if (i_mthd == nonblk_mthd)
        { /* method 2: non-blocking Irecv+Isend+Waitall*/
         if (do_participate[i_pat] == 1)
         { 
          looplng = looplngs[i_msg][i_rep][i_mthd]; 
          for (i_loop = (i_msg < WITH_PRELOOPS ? -PRE_LOOPS : 0); 
               i_loop<looplng; i_loop++)
          {
           if (i_loop == 0) time_beg = MPI_Wtime();
           i_rq = 0; 
           for (i_sr=0; i_sr<sr_count[i_pat]; i_sr++)
           { register char v1=i_sr*i_msg+1, v2=2*v1-1;
                      /* both values must be identical for Msglng==1 */
             register int dest=sr_sranks[i_pat][i_sr];
             register int src =sr_rranks[i_pat][i_sr];
             if ((base+Msglng8) > buflng) base = 0;
             sendbuf[base] = v1; sendbuf[base+Msglng-1] = v2;
             base_vals[i_sr] = base;
#            if (PRINTLEVEL >= 5) 
               printf("[%02d]%4.0fs i_rep=%2d i_msg=%2d i_pat=%2d -- mthd 2 "
                    "loop=%3d sr=%1d before IRcv/Isnd\n", myrank, MPI_Wtime()-all_beg,i_rep, 
                    i_msg, i_pat, i_loop, i_sr); fflush(stdout);
#            endif 
             if (src != MPI_PROC_NULL)
             { MPI_Irecv(recvbuf+base,Msglng,MPI_BYTE, src,
                                  i_sr, comm[i_pat], rq+i_rq); i_rq++; }
             if (dest != MPI_PROC_NULL)
             { MPI_Isend(sendbuf+base,Msglng,MPI_BYTE, dest,
                                  i_sr, comm[i_pat], rq+i_rq); i_rq++; }
             base = base + Msglng8;
           } /* endfor i_sr */
#          if (PRINTLEVEL >= 5) 
              printf("[%02d]%4.0fs i_rep=%2d i_msg=%2d i_pat=%2d -- mthd 2 "
                     "loop=%3d      before Waitall\n", myrank, MPI_Wtime()-all_beg,i_rep, 
                     i_msg, i_pat, i_loop); fflush(stdout);
#          endif 
           MPI_Waitall(i_rq, rq, stss); 
#          if (PRINTLEVEL >= 5) 
             printf("[%02d]%4.0fs i_rep=%2d i_msg=%2d i_pat=%2d -- mthd 2 "
                    "loop=%3d      after  Waitall\n", myrank, MPI_Wtime()-all_beg,i_rep, 
                    i_msg, i_pat, i_loop); fflush(stdout);
#          endif 
           for (i_sr=0; i_sr<sr_count[i_pat]; i_sr++)
           { if ((sr_rranks[i_pat][i_sr] != MPI_PROC_NULL) &&
                 ( (recvbuf[base_vals[i_sr]]
                     != sendbuf[base_vals[i_sr]]) ||
                   (recvbuf[base_vals[i_sr]+Msglng-1]
                     != sendbuf[base_vals[i_sr]+Msglng-1]) ) )
                { printf("ERROR - invalid message content after "
                         "MPI_Irecv+Isend - myrank=%1d i_rep=%1d i_msg=%1d"
                         " i_pat=%1d i_sr=%1d i_loop=%1d Msglng=%1d "
                         "buf=(%1d %1d)!=(%1d %1d)\n",
                         myrank,i_rep,i_msg,i_pat,i_sr,i_loop,Msglng,
                         (int)recvbuf[base_vals[i_sr]],
                         (int)recvbuf[base_vals[i_sr]+Msglng-1],
                         (int)sendbuf[base_vals[i_sr]],
                         (int)sendbuf[base_vals[i_sr]+Msglng-1]);
                  MPI_Abort(MPI_COMM_WORLD,1);
                }
           } /* endfor i_sr */
          } /* endfor i_loop */
          time_end = MPI_Wtime();
         } /* endif do_participate[i_pat] == 1 */
        } /* endif i_mthd==nonblk_mthd */
 
        if (do_participate[i_pat] == 1)
        {
          comm_times[i_msg][i_rep][i_mthd] 
           = comm_times[i_msg][i_rep][i_mthd] + (time_end-time_beg); 
          min_looptimes[i_msg][i_rep][i_mthd] 
           = min(min_looptimes[i_msg][i_rep][i_mthd],time_end-time_beg);
          max_looptimes[i_msg][i_rep][i_mthd] 
           = max(max_looptimes[i_msg][i_rep][i_mthd],time_end-time_beg);
          if (time_end==time_beg) 
          { printf("[%02d]%4.0fs i_rep=%2d i_msg=%2d i_pat=%2d mthd=%1d ERROR: "
             "measurement time == 0\n",myrank, MPI_Wtime()-all_beg,i_rep,i_msg,i_pat,i_mthd);
            bw[i_rep][i_pat][i_msg][i_mthd] = POSINF;
          }else 
            bw[i_rep][i_pat][i_msg][i_mthd] 
             = ((double)(Msglng)) * looplng * sr_count[i_pat] 
               / (time_end-time_beg);
        }else
        { bw[i_rep][i_pat][i_msg][i_mthd] = POSINF;
        } 
#       if (PRINTLEVEL >= 4) 
          printf("[%02d]%4.0fs i_rep=%2d i_msg=%2d i_pat=%2d -- mthd %1d end\n",
                myrank, MPI_Wtime()-all_beg, i_rep, i_msg, i_pat, i_mthd); fflush(stdout);
#       endif 
 
       } /* endif measure[][][][] // endfor i_mthd */ 
 
       /* duplicating the results from best method, if necessary */ 
       if (best_method[i_rep][i_pat] >= 0) 
         for (i_mthd=0; i_mthd<max_methods; i_mthd++)
           bw[i_rep][i_pat][i_msg][i_mthd] 
             = bw[i_rep][i_pat][i_msg][best_method[i_rep][i_pat]]; 
 
      } /* endfor i_pat */ 
      for (i_mthd=0; i_mthd<max_methods; i_mthd++) 
        if (min_looptimes[i_msg][i_rep][i_mthd] == POSINF) 
          min_looptimes[i_msg][i_rep][i_mthd] = 0; 
     } 
     else /* i_rep >= repetitions[i_msg] */
     { 
       /* duplicating the results from rep.0 */
       for (i_pat=0; i_pat<max_patterns; i_pat++)
        for (i_mthd=0; i_mthd<max_methods; i_mthd++)
          bw[i_rep][i_pat][i_msg][i_mthd] = bw[0][i_pat][i_msg][i_mthd];
 
     } /* endif i_rep < repetitions[i_msg] */
	if (pre_msg==1)	/* next loop with 0 */
	{

	  pre_msg = 0; /* not another pre_msg */
	  i_msg = -1;
	}
    } /* endfor i_msg */
  } /* endfor i_rep */
 
  all_end = MPI_Wtime();
  all_sum = all_end - all_beg;

  /* the results of the measurement loop is stored in bw and all_sum */

  /*------------------------------------------------------------------*/
  /*                        reduce to the root                        */
  /*------------------------------------------------------------------*/
 
  MPI_Reduce(&all_sum,&all_delta,1,MPI_DOUBLE,MPI_MAX,0,MPI_COMM_WORLD);

  if (myrank == 0) fprintf(fp,"measurment loop: %7.3f sec\n\n", all_delta);

  MPI_Reduce(bw,minbw,REPETITIONS*PATTERNS*MSGLENGTHS*METHODS,
             MPI_DOUBLE, MPI_MIN, 0, MPI_COMM_WORLD);

  for (i_pat=0; i_pat<max_patterns; i_pat++)
    if ( do_participate[i_pat] != 1)
      for (i_rep=0; i_rep<REPETITIONS; i_rep++)
        for (i_msg=0; i_msg<MSGLENGTHS; i_msg++)
          for (i_mthd=0; i_mthd<max_methods; i_mthd++)
            bw[i_rep][i_pat][i_msg][i_mthd] = 0;

  MPI_Reduce(bw,avgbw,REPETITIONS*PATTERNS*MSGLENGTHS*METHODS,
             MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
  if (myrank == 0)
    for (i_pat=0; i_pat<max_patterns; i_pat++)
      for (i_rep=0; i_rep<REPETITIONS; i_rep++)
        for (i_msg=0; i_msg<MSGLENGTHS; i_msg++)
          for (i_mthd=0; i_mthd<max_methods; i_mthd++)
            avgbw[i_rep][i_pat][i_msg][i_mthd] =
              avgbw[i_rep][i_pat][i_msg][i_mthd] / participate_cnt[i_pat];

  MPI_Reduce(bw,maxbw,REPETITIONS*PATTERNS*MSGLENGTHS*METHODS,
             MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
 
  if (myrank == 0) fprintf(fp,"\nSECTION-LOOP-END\n\n");

 
  /*------------------------------------------------------------------*/
  /*                 printing minbw, avgbw, maxbw                     */
  /*------------------------------------------------------------------*/

  if (myrank == 0)
  { double maxminbw; 
    int i;
 
    fprintf(fp,"\nSECTION-LOOPLNGS-BEGIN\n");
    for (i_mthd=0; i_mthd<max_methods; i_mthd++) 
    {
      fprintf(fp,"\nmethod=%1d  |",i_mthd);
      for (i_rep=0; i_rep<REPETITIONS; i_rep++)
        fprintf(fp," ------- repetition %1d ---------",i_rep); fprintf(fp," [sec]\n");
      fprintf(fp,"(=%7s)|",mthd_description[i_mthd]);
      for (i_rep=0; i_rep<REPETITIONS; i_rep++)
        fprintf(fp," loop   total per pattrn&mthd  "); fprintf(fp,"\n");
      fprintf(fp,"msg length|");
      for (i_rep=0; i_rep<REPETITIONS; i_rep++)
        fprintf(fp,"  lng    time minimum maximum  "); fprintf(fp,"\n");
      fprintf(fp,"----------|");
      for (i_rep=0; i_rep<REPETITIONS; i_rep++)
        fprintf(fp," ---- ------- ------- ---------"); fprintf(fp,"\n");
      for (i_msg=0; i_msg<MSGLENGTHS; i_msg++)
      {
        fprintf(fp," %9d ",msglng[i_msg]);
        for (i_rep=0; i_rep<REPETITIONS; i_rep++)
        {
          fprintf(fp," %4d %7.1e %7.1e %7.1e %c",
                 looplngs[i_msg][i_rep][i_mthd],
                 comm_times[i_msg][i_rep][i_mthd], 
                 min_looptimes[i_msg][i_rep][i_mthd],
                 max_looptimes[i_msg][i_rep][i_mthd],
                 ((i_msg==(MSGLENGTHS-2))&&(best_method[i_rep][0]!= -1)
                  ? 'M' :
                  ((i_msg==(MSGLENGTHS-1))&&(best_method[0][0]!= -1)
                   ? 'R' : ' ')) );
        } 
        fprintf(fp,"\n"); 
      }
      if (best_method[0][0] != -1) 
        fprintf(fp,"\n Explanation: M = only the best method is used\n"
         "              R = only best method and only %1d repetitions\n",
         repetitions[MSGLENGTHS-1]);
      else fprintf(fp,"\n\n\n"); 
    } 
    fprintf(fp,"\nSECTION-LOOPLNGS-END\n\n");
 
    fprintf(fp,"\nSECTION-VALUES-BEGIN\n");
    fprintf(fp,"\nMinimum / Average / Maximum of the bandwidth per node and"
           " on all nodes,\n"
           "printed for all %d repetition (MB/s = 1e6 Bytes/s):\n",
           REPETITIONS); 
    fprintf(fp,"(mthd0=Sendrecv or Send or Recv, mthd1=Alltoallv,"
           " mthd2=Irecv+Isend, pattern see next table)\n\n");
    fprintf(fp," msg length|methd|pat|    ");
    for (i=0; i < REPETITIONS; i++) fprintf(fp," MByte/s");
    fprintf(fp," -> maximum bandw. on all repetitions\n");
    fprintf(fp,"           |     |   |    ");
    for (i=0; i < REPETITIONS; i++) fprintf(fp,"        ");
    fprintf(fp," -> (elapsed time)\n");
 
    all_sum = 0;
    rep_min_sum = 0; 
    for (i_msg=0; i_msg<MSGLENGTHS; i_msg++)
    {
      for (i_mthd=0; i_mthd<max_methods; i_mthd++)
      {
        for (i_pat=0; i_pat<max_patterns; i_pat++)
        {
          if (i_pat==0 && i_mthd==0) fprintf(fp," %10d",msglng[i_msg]);
                                else fprintf(fp,"           ");
          if (i_pat==0) fprintf(fp," mthd%1d",i_mthd);
                   else fprintf(fp,"      ");
          fprintf(fp," p%02d min=",i_pat);
          rep_sum = 0;
          rep_min = POSINF;
          maxminbw = 0;
          for (i_rep=0; i_rep<REPETITIONS; i_rep++)
          { double t;
            if (measure[i_rep][i_pat][i_msg][i_mthd])
            { 
              fprintf(fp," %7.3f",minbw[i_rep][i_pat][i_msg][i_mthd]/1e6);
              if (minbw[i_rep][i_pat][i_msg][i_mthd]>0) 
                t =  msglng[i_msg] / minbw[i_rep][i_pat][i_msg][i_mthd]
                     * (looplngs[i_msg][i_rep][i_mthd] 
                        + (i_msg<WITH_PRELOOPS ? PRE_LOOPS : 0))
                     * sr_count[i_pat];
              else t=999990.0; 
              rep_sum = rep_sum + t;
              rep_min = min(rep_min, t*repetitions[i_msg]);
              maxminbw=max(maxminbw,minbw[i_rep][i_pat][i_msg][i_mthd]);
            }else
            {
              fprintf(fp,"  ------");
            } 
          }
          if (maxminbw != 0)
            fprintf(fp," -> %7.3f (%5.3f s)\n", maxminbw/1e6, rep_sum);
          else
            fprintf(fp," ->  ------ (----- s)\n"); 
          all_sum = all_sum + rep_sum;
          if (rep_min != POSINF) rep_min_sum = rep_min_sum + rep_min; 
          fprintf(fp,"                      avg=");
          for (i_rep=0; i_rep<REPETITIONS; i_rep++)
            if (measure[i_rep][i_pat][i_msg][i_mthd])
              fprintf(fp," %7.3f",avgbw[i_rep][i_pat][i_msg][i_mthd]/1e6);
            else
              fprintf(fp,"   ---- ");
          fprintf(fp,"\n");
          fprintf(fp,"                      max=");
          for (i_rep=0; i_rep<REPETITIONS; i_rep++)
            if (measure[i_rep][i_pat][i_msg][i_mthd])
              fprintf(fp," %7.3f",maxbw[i_rep][i_pat][i_msg][i_mthd]/1e6);
            else
              fprintf(fp,"    --  ");
          fprintf(fp,"\n");
        }
      }
    }
    fprintf(fp,"\nSECTION-VALUES-END\n\n");
    /*all printing to *.short from now*/
    fp = files[1]; 
    fprintf(fp,"\nSECTION-ELAPSED-BEGIN\n");
    fprintf(fp,"\nmeasurment loop: elapsed time               = %7.3f sec\n",
           all_delta);
    fprintf(fp,"\n  sum of max elapsed time per entries above = %7.3f sec\n",
           all_sum); 
    fprintf(fp,"  difference to elapsed time                = %7.3f sec = %3.1f%%\n", 
           all_delta-all_sum, abs(all_delta-all_sum)/all_delta * 100.); 
    fprintf(fp,"\n  sum based on fastest repetition           = %7.3f sec\n",
           rep_min_sum); 
    fprintf(fp,"  difference to elapsed time                = %7.3f sec = %3.1f%%\n", 
           all_delta-rep_min_sum, abs(all_delta-rep_min_sum)/all_delta * 100.); 
    if ( max(abs(all_delta - all_sum),abs(all_delta-rep_min_sum))
         > 0.05 * all_delta )
      fprintf(fp,"\n  CAUTION: A difference above is more than 5 %%."
             " There may be problems with the MPI implementation or"
             " processes weren't on dedicated processors\n");
    else 
      fprintf(fp,"\n  The difference is less than 5 %%\n");
    fprintf(fp,"\nSECTION-ELAPSED-END\n\n");
  }

 
  /*------------------------------------------------------------------*/
  /*                   printing the pattern details                   */
  /*------------------------------------------------------------------*/

  if (myrank == 0)
  { 
    fprintf(fp,"\nSECTION-PATTERN-BEGIN\n\n");
    fprintf(fp,"\nPattern parameters: sendrecv  total number  messages/  messages/  unused  best method\n"
             "                       calls   of messages  used node  node&call   nodes  per repetition\n"
             "---- ----------------- -----  ------------  ---------  ---------  ------  --------------\n");
    for (i_pat=0; i_pat<max_patterns; i_pat++)
    { 
      fprintf(fp," p%02d %-17s %5d  %12d  %9.2f  %9.2f  %6d  (",
             i_pat, pat_description[i_pat], sr_count[i_pat], 
             all_msg_cnt[i_pat], 
             ((double)all_msg_cnt[i_pat]) / participate_cnt[i_pat],
             ((double)all_msg_cnt[i_pat]) / 
               (participate_cnt[i_pat]*sr_count[i_pat]),
             size-participate_cnt[i_pat] );  
      for (i_rep=0; i_rep<REPETITIONS; i_rep++)
        fprintf(fp," %2d",best_method[i_rep][i_pat]);  fprintf(fp," )\n");
    } 
    fprintf(fp,"\nSECTION-PATTERN-END\n\n");
  } 

 
  /*------------------------------------------------------------------*/
  /*          analyzing the measurements and computing b_eff          */
  /*------------------------------------------------------------------*/

  if (myrank == 0)
  { 
    /* official analysis path: */ 
    /* logavg_pat ( avg_L ( max_mthd ( max_rep ( b(L) ) ) ) ) */
    for (i_pat=0; i_pat<max_patterns; i_pat++)
    {
      for (accum_bw = 0, i_msg=0; i_msg<MSGLENGTHS; i_msg++)
      {
        for (maxmthd_bw = 0, i_mthd=0; i_mthd<max_methods; i_mthd++)
        {
          for (maxrep_bw = 0, i_rep=0; i_rep<REPETITIONS; i_rep++)
          { 
            maxrep_bw = max( maxrep_bw, 
                              minbw[i_rep][i_pat][i_msg][i_mthd]  
                               * all_msg_cnt[i_pat] / sr_count[i_pat] );
          }
          maxmthd_bw = max( maxmthd_bw, maxrep_bw ); 
        }
        b_eff_pat_msg[i_msg][i_pat] = maxmthd_bw; 
        accum_bw = accum_bw + maxmthd_bw; 
      }
      avgL_bw = accum_bw / MSGLENGTHS; 
      b_eff_pat_msg[MSGLENGTHS][i_pat] = avgL_bw; 
    }
    for(i_msg=0; i_msg<MSGLENGTHS+1; i_msg++)
      logavg_pat( /*IN*/  b_eff_pat_msg[i_msg],
                  /*OUT*/ &(b_eff_rings_msg[i_msg]),
                          &(b_eff_random_msg[i_msg]),
                          &(b_eff_final_msg[i_msg])); 
 
    /* 2nd analysis path only as information: */ 
    /* logavg_pat ( max_mthd ( avg_L ( max_rep ( b(L) ) ) ) ) */
    for (i_pat=0; i_pat<max_patterns; i_pat++)
    {
      for (maxmthd_bw = 0, i_mthd=0; i_mthd<max_methods; i_mthd++)
      {
        for (accum_bw = 0, i_msg=0; i_msg<MSGLENGTHS; i_msg++)
        {
          for (maxrep_bw = 0, i_rep=0; i_rep<REPETITIONS; i_rep++)
          { 
            maxrep_bw = max( maxrep_bw, 
                              minbw[i_rep][i_pat][i_msg][i_mthd]  
                               * all_msg_cnt[i_pat] / sr_count[i_pat] );
          }
          if ((i_pat==one_pingpong_pair_pat) && (i_mthd==sndrcv_mthd))
          {
            if ((i_msg==0) && (maxrep_bw>0)) 
              latency_onePingPong = ((double)all_msg_cnt[i_pat])
                                    / maxrep_bw / sr_count[i_pat];
            if (i_msg==(MSGLENGTHS-1)) b_asymp_onePingPong = maxrep_bw;
          }
          accum_bw = accum_bw + maxrep_bw; 
        }
        avgL_bw = accum_bw / MSGLENGTHS; 
        maxmthd_bw = max( maxmthd_bw, avgL_bw); 
        b_eff_pat_mthd[i_mthd][i_pat] = avgL_bw;
      }
      b_eff_pat_mthd[max_methods][i_pat] = maxmthd_bw;
    }
    for(i_mthd=0; i_mthd<max_methods+1; i_mthd++)
      logavg_pat( /*IN*/  b_eff_pat_mthd[i_mthd],
                  /*OUT*/ &(b_eff_rings_mthd[i_mthd]),
                          &(b_eff_random_mthd[i_mthd]),
                          &(b_eff_final_mthd[i_mthd])); 
 
    /* 3rd analysis path only as information: */ 
    /* logavg_pat ( max_rep ( avg_L ( max_mthd ( b(L) ) ) ) ) */
    for (i_pat=0; i_pat<max_patterns; i_pat++)
    {
      for (maxrep_bw = 0, i_rep=0; i_rep<REPETITIONS; i_rep++)
      { 
        for (accum_bw = 0, i_msg=0; i_msg<MSGLENGTHS; i_msg++)
        {
          for (maxmthd_bw = 0, i_mthd=0; i_mthd<max_methods; i_mthd++)
          {
            maxmthd_bw = max( maxmthd_bw, 
                              minbw[i_rep][i_pat][i_msg][i_mthd]  
                               * all_msg_cnt[i_pat] / sr_count[i_pat] );
          }
          accum_bw = accum_bw + maxmthd_bw; 
        }
        avgL_bw = accum_bw / MSGLENGTHS; 
        maxrep_bw = max( maxrep_bw, avgL_bw ); 
        b_eff_pat_rep[i_rep][i_pat] = avgL_bw; 
      }
      b_eff_pat_rep[REPETITIONS][i_pat] = maxrep_bw; 
    }
    for(i_rep=0; i_rep<REPETITIONS+1; i_rep++)
      logavg_pat( /*IN*/  b_eff_pat_rep[i_rep],
                  /*OUT*/ &(b_eff_rings_rep[i_rep]),
                          &(b_eff_random_rep[i_rep]),
                          &(b_eff_final_rep[i_rep])); 
 
    /* 4th analysis path only as information: */ 
    /* for all methods: logavg_pat ( avg_L ( max_rep ( b(L) ) ) ) */
    for (i_mthd=0; i_mthd<max_methods; i_mthd++)
    { 
      for (i_pat=0; i_pat<max_patterns; i_pat++)
      {
        for (accum_bw = 0, i_msg=0; i_msg<MSGLENGTHS; i_msg++)
        {
          for (maxrep_bw = 0, i_rep=0; i_rep<REPETITIONS; i_rep++)
          { 
            maxrep_bw = max( maxrep_bw, 
                              minbw[i_rep][i_pat][i_msg][i_mthd]  
                               * all_msg_cnt[i_pat] / sr_count[i_pat] );
          }
          b_eff_pat_msg_mthd[i_msg][i_mthd][i_pat] = maxrep_bw; 
          accum_bw = accum_bw + maxrep_bw; 
        }
        avgL_bw = accum_bw / MSGLENGTHS; 
        b_eff_pat_msg_mthd[MSGLENGTHS][i_mthd][i_pat] = avgL_bw; 
      }
      for(i_msg=0; i_msg<MSGLENGTHS+1; i_msg++)
        logavg_pat( /*IN*/  b_eff_pat_msg_mthd[i_msg][i_mthd],
                    /*OUT*/ &(b_eff_rings_msg_mthd[i_msg][i_mthd]),
                            &(b_eff_random_msg_mthd[i_msg][i_mthd]),
                            &(b_eff_final_msg_mthd[i_msg][i_mthd])); 
    }
  } 

 
  /*------------------------------------------------------------------*/
  /*                  printing the analysis results                   */
  /*------------------------------------------------------------------*/
 
  if (myrank == 0)
  { 
    fprintf(fp,"\nPrinting accumulated bandwidth / number of processes\n"
             "         = bandwidth per process   (in MBytes/sec = 1e6 Bytes/sec)\n"
             "Additionally the accumulated value processes is printed in the last column\n"
           "\nOnly the cyclic and random patterns are used to compute b_eff\n"
             "(except p00), the other patterns are only as information.\n");  

    /* ----------- printing info: pattern x methods --------- */
    fprintf(fp,"\nSECTION-BY-METHODS-BEGIN\n\n");
    fprintf(fp,"\n2nd analysis path, only as information, last row:\n"
           "for all mthd: logavg_pat ( avg_L ( max_rep ( b(L) ) ) )\n"
           "|| logavg_pat ( max_mthd ( avg_L ( max_rep ( b(L) ) ) ) )\n");
    fprintf(fp,"\n pattern   /    methods:");
    for (i_mthd=0; i_mthd<max_methods; i_mthd++)
      fprintf(fp," %7s",mthd_description[i_mthd]);
    fprintf(fp," -> best method -> accumulated\n\n"); 
    for (i_pat=0; i_pat<max_patterns; i_pat++)
    {
      if (i_pat==(MAX_RINGS+MAX_RANDOM)) fprintf(fp,
       " -- additional patterns that are not used to compute b_eff:\n");
      fprintf(fp," p%02d %-17s :", i_pat, pat_description[i_pat]);
      for (i_mthd=0; i_mthd<max_methods; i_mthd++)
        fprintf(fp," %7.3f", b_eff_pat_mthd[i_mthd][i_pat]/1e6/size);
      fprintf(fp," -> %7.3f -> %10.3f MByte/s\n",b_eff_pat_mthd[max_methods][i_pat]/1e6/size, b_eff_pat_mthd[max_methods][i_pat]/1e6);
    } 
    fprintf(fp,"\n log_avg of all rings  :"); 
    for (i_mthd=0; i_mthd<max_methods; i_mthd++)
      fprintf(fp," %7.3f", b_eff_rings_mthd[i_mthd]/1e6/size);
    fprintf(fp," || %7.3f -> %10.3f MByte/s\n",b_eff_rings_mthd[max_methods]/1e6/size, b_eff_rings_mthd[max_methods]/1e6);
    fprintf(fp," log_avg of all random :"); 
    for (i_mthd=0; i_mthd<max_methods; i_mthd++)
      fprintf(fp," %7.3f", b_eff_random_mthd[i_mthd]/1e6/size);
    fprintf(fp," || %7.3f -> %10.3f MByte/s\n",b_eff_random_mthd[max_methods]/1e6/size, b_eff_random_mthd[max_methods]/1e6);
    fprintf(fp," log_avg(ring,random)  :"); 
    for (i_mthd=0; i_mthd<max_methods; i_mthd++)
      fprintf(fp," %7.3f", b_eff_final_mthd[i_mthd]/1e6/size);
    fprintf(fp," ||(%7.3f -> %10.3f)MByte/s\n",b_eff_final_mthd[max_methods]/1e6/size, b_eff_final_mthd[max_methods]/1e6);
    fprintf(fp,"  * size ->\n accumulated on all pr.:"); 
    for (i_mthd=0; i_mthd<max_methods; i_mthd++)
      fprintf(fp," %7.3f", b_eff_final_mthd[i_mthd]/1e6);
    fprintf(fp," ||(%7.3f)MByte/s\n",b_eff_final_mthd[max_methods]/1e6);
    if (best_method[0][0] != -1) fprintf(fp,CAUTION);
    fprintf(fp,"\nSECTION-BY-METHODS-END\n\n");
       

    /* ----------- printing info: pattern x repetitions ----- */
    fprintf(fp,"\nSECTION-BY-REPETITIONS-BEGIN\n\n");
    fprintf(fp,"\n3rd analysis path, only as information, last row:\n"
           "for all rep.:logavg_pat ( avg_L ( max_mthd ( b(L) ) ) )\n"
           "|| logavg_pat ( max_rep ( avg_L ( max_mthd ( b(L) ) ) ) )\n");
    fprintf(fp,"\n pattern  /  repetition:");
    for(i_rep=0;i_rep<REPETITIONS;i_rep++) fprintf(fp,"   rep.%1d",i_rep);
    fprintf(fp," -> best repetition -> accumulated\n\n");
    for (i_pat=0; i_pat<max_patterns; i_pat++)
    {
      if (i_pat==(MAX_RINGS+MAX_RANDOM)) fprintf(fp,
       " -- additional patterns that are not used to compute b_eff:\n");
      fprintf(fp," p%02d %-17s :", i_pat, pat_description[i_pat]);
      for (i_rep=0; i_rep<REPETITIONS; i_rep++)
        fprintf(fp," %7.3f", b_eff_pat_rep[i_rep][i_pat]/1e6/size);
      fprintf(fp," -> %7.3f -> %10.3f MByte/s\n",b_eff_pat_rep[REPETITIONS][i_pat]/1e6/size, b_eff_pat_rep[REPETITIONS][i_pat]/1e6);
    } 
    fprintf(fp,"\n log_avg of all rings  :"); 
    for (i_rep=0; i_rep<REPETITIONS; i_rep++)
      fprintf(fp," %7.3f", b_eff_rings_rep[i_rep]/1e6/size);
    fprintf(fp," || %7.3f -> %10.3f MByte/s\n",b_eff_rings_rep[REPETITIONS]/1e6/size, b_eff_rings_rep[REPETITIONS]/1e6);
    fprintf(fp," log_avg of all random :"); 
    for (i_rep=0; i_rep<REPETITIONS; i_rep++)
      fprintf(fp," %7.3f", b_eff_random_rep[i_rep]/1e6/size);
    fprintf(fp," || %7.3f -> %10.3f MByte/s\n",b_eff_random_rep[REPETITIONS]/1e6/size, b_eff_random_rep[REPETITIONS]/1e6);
    fprintf(fp," log_avg(ring,random)  :"); 
    for (i_rep=0; i_rep<REPETITIONS; i_rep++)
      fprintf(fp," %7.3f", b_eff_final_rep[i_rep]/1e6/size);
    fprintf(fp," ||(%7.3f -> %10.3f)MByte/s\n",b_eff_final_rep[REPETITIONS]/1e6/size, b_eff_final_rep[REPETITIONS]/1e6);
    fprintf(fp,"  * size ->\n accumulated on all pr.:"); 
    for (i_rep=0; i_rep<REPETITIONS; i_rep++)
      fprintf(fp," %7.3f", b_eff_final_rep[i_rep]/1e6);
    fprintf(fp," ||(%7.3f)MByte/s\n",b_eff_final_rep[REPETITIONS]/1e6);
    fprintf(fp,"\nSECTION-BY-REPETITIONS-END\n\n");

       
    /* ----------- printing info: pattern&method x length --- */
    fprintf(fp,"\nSECTION-BY-MTHD-MSGLNG-BEGIN\n\n");
    fprintf(fp,"\n4th analysis path, only as information, last 3 rows:\n"
           "for all methods and some msglng L: logavg_pat ( max_rep ( b(L) ) )\n"
           "for all methods: logavg_pat ( avg_L ( max_rep ( b(L) ) ) )\n");
    fprintf(fp,"\n pattern & method / msg:         ");
    for(i_msg=0;i_msg<MSGLENGTHS;i_msg+=4) fprintf(fp," %7d",msglng[i_msg]);
    fprintf(fp," -> average -> accumulated\n");
    fprintf(fp,"                        (latency of one sendrecv (or equivalent method) in microsec)\n");
    for (i_pat=0; i_pat<max_patterns; i_pat++)
    {
      if (i_pat==(MAX_RINGS+MAX_RANDOM)) fprintf(fp,
       " -- additional patterns that are not used to compute b_eff:\n");
      fprintf(fp," p%02d %-17s\n", i_pat, pat_description[i_pat]);
      for (i_mthd=0; i_mthd<max_methods; i_mthd++)
      { fprintf(fp," p%02d method %1d =%-7s :",
                   i_pat, i_mthd, mthd_description[i_mthd]);
        if (b_eff_pat_msg_mthd[0][i_mthd][i_pat]>0) 
         fprintf(fp,"(%7.3f)",1e6 * ((double)all_msg_cnt[i_pat])
                                  / b_eff_pat_msg_mthd[0][i_mthd][i_pat]
                                  / sr_count[i_pat] );
        else fprintf(fp,"(-------)"); 
        for (i_msg=0; i_msg<MSGLENGTHS; i_msg+=4)
          fprintf(fp," %7.3f", b_eff_pat_msg_mthd[i_msg][i_mthd][i_pat]/1e6/size);
        fprintf(fp," -> %7.3f -> %10.3f MByte/s\n",b_eff_pat_msg_mthd[MSGLENGTHS][i_mthd][i_pat]/1e6/size, b_eff_pat_msg_mthd[MSGLENGTHS][i_mthd][i_pat]/1e6);
      } 
    } 
    fprintf(fp,"\n log_avg of all rings  \n"); 
    for (i_mthd=0; i_mthd<max_methods; i_mthd++)
    { fprintf(fp," - ring,      method %1d = %-7s:",i_mthd, mthd_description[i_mthd]);
      for (i_msg=0; i_msg<MSGLENGTHS; i_msg+=4)
        fprintf(fp," %7.3f", b_eff_rings_msg_mthd[i_msg][i_mthd]/1e6/size);
      fprintf(fp," || %7.3f -> %10.3f MByte/s\n",b_eff_rings_msg_mthd[MSGLENGTHS][i_mthd]/1e6/size, b_eff_rings_msg_mthd[MSGLENGTHS][i_mthd]/1e6);
    }
    fprintf(fp," log_avg of all random \n"); 
    for (i_mthd=0; i_mthd<max_methods; i_mthd++)
    { fprintf(fp," - random,    method %1d = %-7s:",i_mthd, mthd_description[i_mthd]);
      for (i_msg=0; i_msg<MSGLENGTHS; i_msg+=4)
        fprintf(fp," %7.3f", b_eff_random_msg_mthd[i_msg][i_mthd]/1e6/size);
      fprintf(fp," || %7.3f -> %10.3f MByte/s\n",b_eff_random_msg_mthd[MSGLENGTHS][i_mthd]/1e6/size, b_eff_random_msg_mthd[MSGLENGTHS][i_mthd]/1e6);
    }
    fprintf(fp," log_avg(ring,random)  \n"); 
    for (i_mthd=0; i_mthd<max_methods; i_mthd++)
    { fprintf(fp," - average,   method %1d = %-7s:",i_mthd, mthd_description[i_mthd]);
      for (i_msg=0; i_msg<MSGLENGTHS; i_msg+=4)
        fprintf(fp," %7.3f", b_eff_final_msg_mthd[i_msg][i_mthd]/1e6/size);
      fprintf(fp," || %7.3f -> %10.3f MByte/s\n",b_eff_final_msg_mthd[MSGLENGTHS][i_mthd]/1e6/size, b_eff_final_msg_mthd[MSGLENGTHS][i_mthd]/1e6);
    }
    fprintf(fp,"  * size ->\n accumulated on all processes:\n"); 
    for (i_mthd=0; i_mthd<max_methods; i_mthd++)
    { fprintf(fp," - accumulated, mthd %1d = %-7s:",i_mthd, mthd_description[i_mthd]);
      for (i_msg=0; i_msg<MSGLENGTHS; i_msg+=4)
        fprintf(fp," %7.3f", b_eff_final_msg_mthd[i_msg][i_mthd]/1e6);
      fprintf(fp," || %7.3f MByte/s\n",b_eff_final_msg_mthd[MSGLENGTHS][i_mthd]/1e6);
    }
    if (best_method[0][0] != -1) fprintf(fp,CAUTION);
    fprintf(fp,"\nSECTION-BY-MTHD-MSGLNG-END\n\n");

       
    /* ----------- printing info: length * pattern and methods ----- */
    fprintf(fp,"\nSECTION-BY-MSGLNG-BEGIN\n\n");
    fprintf(fp,"\n5th analysis path only as information:\n"
           "logavg_pat ( max_mthd ( max_rep ( b(L) ) ) )\n"
           "and for all methods: logavg_pat ( max_rep ( b(L) ) )\n");
    fprintf(fp,"\nmsg length|accumulated| effective bandwidth per process:\n"
               "          |  effective| average    rings   random");
    for (i_mthd=0; i_mthd<max_methods; i_mthd++)
      fprintf(fp," method_%1d",i_mthd); 
    fprintf(fp,"\n");
    fprintf(fp,"          |  bandwidth| crt,rnd     only     only");
    for (i_mthd=0; i_mthd<max_methods; i_mthd++)
      fprintf(fp,"  %7s",mthd_description[i_mthd]); 
    fprintf(fp,"\n");
    for (i_msg=0; i_msg<MSGLENGTHS; i_msg++)
    {
      fprintf(fp," %9d ",msglng[i_msg]);
      fprintf(fp," %10.3f", b_eff_final_msg[i_msg]/1e6);
      fprintf(fp," %8.3f", b_eff_final_msg[i_msg]/1e6/size);
      fprintf(fp," %8.3f", b_eff_rings_msg[i_msg]/1e6/size);
      fprintf(fp," %8.3f", b_eff_random_msg[i_msg]/1e6/size);
      for (i_mthd=0; i_mthd<max_methods; i_mthd++)
        fprintf(fp," %8.3f", b_eff_final_msg_mthd[i_msg][i_mthd]/1e6/size);
      fprintf(fp,"\n"); 
    } 
    if (best_method[0][0] != -1) fprintf(fp,CAUTION);
    fprintf(fp,"\nSECTION-BY-MSGLNG-END\n\n");


    /* ----------- printing official: pattern x length ------ */
    fprintf(fp,"\nSECTION-BY-PATTERN-MSGLNG-BEGIN\n\n");
    fprintf(fp,"\nofficial (1st) analysis path -- see last column:\n"
           "(only columns of some message lengths are printed)\n" 
           "logavg_pat ( avg_L ( max_mthd ( max_rep ( b(L) ) ) ) )\n");
    fprintf(fp,"\n pattern  /  msg-length:         ");
    for(i_msg=0;i_msg<MSGLENGTHS;i_msg+=4) fprintf(fp," %7d",msglng[i_msg]);
    fprintf(fp," -> average -> accumulated\n");
    fprintf(fp,"                        (latency of one sendrecv (or equivalent method) in microsec)\n");
    for (i_pat=0; i_pat<max_patterns; i_pat++)
    {
      if (i_pat==(MAX_RINGS+MAX_RANDOM)) fprintf(fp,
       " -- additional patterns that are not used to compute b_eff:\n");
      fprintf(fp," p%02d %-17s :", i_pat, pat_description[i_pat]);
      if (b_eff_pat_msg[0][i_pat]>0) 
        fprintf(fp,"(%7.3f)",1e6 * ((double)all_msg_cnt[i_pat])
                                 / b_eff_pat_msg[0][i_pat]
                                 / sr_count[i_pat] );
      else fprintf(fp,"(-------)"); 
      for (i_msg=0; i_msg<MSGLENGTHS; i_msg+=4)
        fprintf(fp," %7.3f", b_eff_pat_msg[i_msg][i_pat]/1e6/size);
      fprintf(fp," -> %7.3f -> %10.3f MByte/s\n",b_eff_pat_msg[MSGLENGTHS][i_pat]/1e6/size, b_eff_pat_msg[MSGLENGTHS][i_pat]/1e6);
    } 
    fprintf(fp,"\n log_avg of all rings  :         "); 
    for (i_msg=0; i_msg<MSGLENGTHS; i_msg+=4)
      fprintf(fp," %7.3f", b_eff_rings_msg[i_msg]/1e6/size);
    fprintf(fp," || %7.3f -> %10.3f MByte/s\n",b_eff_rings_msg[MSGLENGTHS]/1e6/size, b_eff_rings_msg[MSGLENGTHS]/1e6);
    fprintf(fp," log_avg of all random :         "); 
    for (i_msg=0; i_msg<MSGLENGTHS; i_msg+=4)
      fprintf(fp," %7.3f", b_eff_random_msg[i_msg]/1e6/size);
    fprintf(fp," || %7.3f -> %10.3f MByte/s\n",b_eff_random_msg[MSGLENGTHS]/1e6/size, b_eff_random_msg[MSGLENGTHS]/1e6);
    fprintf(fp," log_avg(ring,random)  :         "); 
    for (i_msg=0; i_msg<MSGLENGTHS; i_msg+=4)
      fprintf(fp," %7.3f", b_eff_final_msg[i_msg]/1e6/size);
    fprintf(fp," || %7.3f -> %10.3f MByte/s\n",b_eff_final_msg[MSGLENGTHS]/1e6/size, b_eff_final_msg[MSGLENGTHS]/1e6);
    fprintf(fp,"  * size ->\n accumulated on all pr.:         "); 
    for (i_msg=0; i_msg<MSGLENGTHS; i_msg+=4)
      fprintf(fp," %7.3f", b_eff_final_msg[i_msg]/1e6);
    fprintf(fp," || %7.3f MByte/s\n",b_eff_final_msg[MSGLENGTHS]/1e6);
    fprintf(fp,"\nSECTION-BY-PATTERN-MSGLNG-END\n\n");
 
    fprintf(fp,"\nSECTION-BEFF-BEGIN\n\n"); /*Summary, Machine Information*/

    b_eff = b_eff_final_msg[MSGLENGTHS];

    fprintf(fp,"\nThe effective bandwidth is b_eff = %10.3f MByte/s on %1d processes\n",
           b_eff/1e6, size);
    fprintf(fp,  "                               ( = %10.3f MByte/s  * %1d processes)\n",
           b_eff/1e6/size,size);    
    fprintf(fp,"Latency: %8.3f microsec\n\n",1/(b_eff_final_msg[0]/1e6/size));

    fprintf(fp,"Lmax: %5.0f MB\n",Lmax/(1024.*1024.));
    fprintf(fp,"b_eff at Lmax: %10.3f MByte/s on %1d processes\n", 
	   b_eff_final_msg[20]/1e6,size);
    fprintf(fp,"           ( : %10.3f MByte/s * %1d processes)\n\n",
	   b_eff_final_msg[20]/1e6/size,size);
   
    fprintf(fp,"b_eff at Lmax (ring pattern):  %10.3f MByte/s on %1d processes\n",
	    b_eff_rings_msg[20]/1e6,size);
    fprintf(fp,"                          ( :  %10.3f MByte/s * %1d processes)\n",
	    b_eff_rings_msg[20]/1e6/size,size);
    fprintf(fp,"Latency ring pattern:           %8.3f microsec\n",(1/b_eff_rings_msg[0])*1e6);

    if (latency_onePingPong > 0)
      fprintf(fp,"Ping-pong latency:            %8.3f microsec\n",latency_onePingPong*1e6);
    if (b_asymp_onePingPong > 0) 
      fprintf(fp,"Ping-pong bandwidth at Lmax: %10.3f MByte/s at Lmax= %6.1f MB\n\n (MByte/s=1e6 Byte/s)  (MB=2**20 Byte)\n",
	      b_asymp_onePingPong/1e6, Lmax/(1024.*1024.));  

    for (i=1;i<=2;i++)
    {
      fprintf(files[i],"system parameters  : %1d nodes, %1d MB/node\n",
             size, MEMORY_PER_PROCESSOR); 
      uname(&uname_info);
      fprintf(files[i],"       system name : %s\n",   uname_info.sysname); 
      fprintf(files[i],"       hostname    : %s\n",   uname_info.nodename); 
      fprintf(files[i],"       OS release  : %s\n",   uname_info.release); 
      fprintf(files[i],"       OS version  : %s\n",   uname_info.version); 
      fprintf(files[i],"       machine     : %s\n\n", uname_info.machine); 
 
      t_now = time((time_t *) 0); 
      fprintf(files[i],"Date of measurement: %s\n", ctime(&t_now)); 
     
      total_end = MPI_Wtime();
      fprintf(files[i],"Total execution wall clock time = %5.0f seconds\n\n",total_end - total_beg); 
    
      fprintf(files[i],"            | number  |   b_eff   |  Lmax  |   b_eff   |   b_eff   | Latency | Latency | Latency | ping-pong  \n");
      fprintf(files[i],"            | of pro  |           |        |  at Lmax  |  at Lmax  | rings & |  rings  |  ping-  | bandwith   \n");
      fprintf(files[i],"            | cessors |           |        |  rings &  |   rings   | random  |  only   |  pong   |            \n");
      fprintf(files[i],"            |         |           |        |  random   |   only    | micro-  | micro-  | micro-  |            \n");
      fprintf(files[i],"            |         |  MByte/s  |        |  MByte/s  |  MByte/s  |  sec    |  sec    |  sec    |  MByte/s   \n");
      fprintf(files[i],"--------------------------------------------------------------------------------------------------------------\n");
      fprintf(files[i],"            |                                                                                                 \n");
      fprintf(files[i],"accumulated |  %5d    %9.0f   %3.0f MB  %9.0f    %9.0f    %6.3f    %6.3f    %6.3f    %9.0f                    \n",
              size, b_eff/1e6, Lmax/(1024.*1024.), b_eff_final_msg[20]/1e6, b_eff_rings_msg[20]/1e6, 1/(b_eff_final_msg[0]/1e6/size),
              1/(b_eff_rings_msg[0]/1e6/size), latency_onePingPong*1e6, b_asymp_onePingPong/1e6);
      fprintf(files[i],"            |                                                                                                 \n");
      fprintf(files[i],"per process |           %9.0f           %9.0f    %9.0f                                                      \n\n",
              b_eff/1e6/size, b_eff_final_msg[20]/1e6/size, b_eff_rings_msg[20]/1e6/size);
    
      fprintf(files[i],"\nSECTION-BEFF-END\n\n");
    }

    for (i=0; i<=2; i++){
      fprintf(files[i],"b_eff%s = %10.3f MB/s = %7.3f * %3d PEs with %3d MB/PE on %s %s %s %s %s\n",
              TSTR, b_eff/1e6, b_eff/1e6/size, size, MEMORY_PER_PROCESSOR,
              uname_info.sysname, uname_info.nodename, uname_info.release,
              uname_info.version, uname_info.machine);
    }
 
    fprintf(stderr,
           "b_eff%s = %10.3f MB/s = %7.3f * %3d PEs with %3d MB/PE on %s %s %s %s %s\n",
           TSTR, b_eff/1e6, b_eff/1e6/size, size, MEMORY_PER_PROCESSOR,
           uname_info.sysname, uname_info.nodename, uname_info.release,
           uname_info.version, uname_info.machine); 
 
    if (files[0]!=stdout)
    {
      fclose(files[0]);
	  fclose(files[1]);
	  fclose(files[2]);
    }

#ifndef SHORT
#  ifndef TEST

  /*--------------------------------------------------------------------------*/
  /* Create result.plot  -  use it with gnuplot                               */
  /*                                                                          */
    fprintf(plot,"# b_eff %s - Use this file with Gnuplot\n#\n", Revision);
    fprintf(plot,"#\n#\n"); 

    for (i_mthd=0; i_mthd<=2; i_mthd++)
    {
      fprintf(plot,"\n\n# %s: ring & random\n", mthd_description[i_mthd]);
      for (i_msg=0; i_msg<=20; i_msg++)
      {  
	if ((best_method[0][0] != -1) && (i_msg == (MSGLENGTHS-2))) break;
        fprintf(plot,"%10i ", msglng[i_msg]);
        for (i_pat=0; i_pat<=5; i_pat++)
          fprintf(plot,"%7.3f ", b_eff_pat_msg_mthd[i_msg][i_mthd][i_pat]/1e6/size);

        /*worst random, best random, logavg random*/
        random_min = b_eff_pat_msg_mthd[i_msg][i_mthd][6];
        random_max = b_eff_pat_msg_mthd[i_msg][i_mthd][6];
        for (i_pat=7; i_pat<=35; i_pat++)
        {
          if (b_eff_pat_msg_mthd[i_msg][i_mthd][i_pat] < random_min) random_min = b_eff_pat_msg_mthd[i_msg][i_mthd][i_pat];
          if (b_eff_pat_msg_mthd[i_msg][i_mthd][i_pat] > random_max) random_max = b_eff_pat_msg_mthd[i_msg][i_mthd][i_pat];
        }
        fprintf(plot,"%7.3f %7.3f %7.3f ", random_min/1e6/size, b_eff_random_msg_mthd[i_msg][i_mthd]/1e6/size, random_max/1e6/size);
        fprintf(plot,"\n");
      }

      fprintf(plot,"\n\n# %s: additional\n", mthd_description[i_mthd]);
      for (i_msg=0; i_msg<=20; i_msg++)
      {
	if ((best_method[0][0] != -1) && (i_msg == (MSGLENGTHS-2))) break;
        fprintf(plot,"%10i ", msglng[i_msg]);
        for (i_pat=36; i_pat<=38; i_pat++)
        fprintf(plot,"%7.3f ", b_eff_pat_msg_mthd[i_msg][i_mthd][i_pat]/1e6/size);
        for (i_pat=40; i_pat<=45; i_pat++)
        fprintf(plot,"%7.3f ", b_eff_pat_msg_mthd[i_msg][i_mthd][i_pat]/1e6/size);
        fprintf(plot,"\n");
      }
    }
		
    fprintf(plot,"\n\n# best: ring & random\n", mthd_description[i_mthd]);
    for (i_msg=0; i_msg<=20; i_msg++)
    {  
      fprintf(plot,"%10i ", msglng[i_msg]);
      for (i_pat=0; i_pat<=5; i_pat++)
        fprintf(plot,"%7.3f ", b_eff_pat_msg[i_msg][i_pat]/1e6/size);

      /*worst random, best random, logavg random*/
      random_min = b_eff_pat_msg[i_msg][6];
      random_max = b_eff_pat_msg[i_msg][6];
      for (i_pat=7; i_pat<=35; i_pat++)
      {
        if (b_eff_pat_msg[i_msg][i_pat] < random_min) random_min = b_eff_pat_msg[i_msg][i_pat];
        if (b_eff_pat_msg[i_msg][i_pat] > random_max) random_max = b_eff_pat_msg[i_msg][i_pat];
      }
      fprintf(plot,"%7.3f %7.3f %7.3f ", random_min/1e6/size, b_eff_random_msg[i_msg]/1e6/size, random_max/1e6/size);
      fprintf(plot,"\n");
    }
      
    fprintf(plot,"\n\n# best: additional\n", mthd_description[i_mthd]);
    for (i_msg=0; i_msg<=20; i_msg++)
    {
      fprintf(plot,"%10i ", msglng[i_msg]);
      for (i_pat=36; i_pat<=38; i_pat++)
        fprintf(plot,"%7.3f ", b_eff_pat_msg[i_msg][i_pat]/1e6/size);
      for (i_pat=40; i_pat<=45; i_pat++)
        fprintf(plot,"%7.3f ", b_eff_pat_msg[i_msg][i_pat]/1e6/size);
      fprintf(plot,"\n");
    }	

    /* 9. compare the methods*/
    fprintf(plot,"\n\n# compare the methods: ring & random\n");
    for (i_msg=0; i_msg<=20; i_msg++)
    {
      if ((best_method[0][0] != -1) && (i_msg == (MSGLENGTHS-2))) break;
      fprintf(plot,"%10i ", msglng[i_msg]);
      fprintf(plot,"%7.3f ", b_eff_rings_msg_mthd[i_msg][0]/1e6/size);
      fprintf(plot,"%7.3f ", b_eff_rings_msg_mthd[i_msg][1]/1e6/size);
      fprintf(plot,"%7.3f ", b_eff_rings_msg_mthd[i_msg][2]/1e6/size);
      fprintf(plot,"%7.3f ", b_eff_random_msg_mthd[i_msg][0]/1e6/size);
      fprintf(plot,"%7.3f ", b_eff_random_msg_mthd[i_msg][1]/1e6/size);
      fprintf(plot,"%7.3f ", b_eff_random_msg_mthd[i_msg][2]/1e6/size);
      fprintf(plot,"\n");
    }

    /* 10. compare ring- and random-pattern*/
    fprintf(plot,"\n\n# compare ring & random\n");
    for (i_msg=0; i_msg<=20; i_msg++)
    {  
      fprintf(plot,"%10i ", msglng[i_msg]);

      /*worst ring, best ring, logavg rings*/
      ring_min = b_eff_pat_msg[i_msg][0];
      ring_max = b_eff_pat_msg[i_msg][0];
      for (i_pat=1; i_pat<=5; i_pat++)
      {
        if (b_eff_pat_msg[i_msg][i_pat] < ring_min) ring_min = b_eff_pat_msg[i_msg][i_pat];
        if (b_eff_pat_msg[i_msg][i_pat] > ring_max) ring_max = b_eff_pat_msg[i_msg][i_pat];
      }
      fprintf(plot,"%7.3f %7.3f %7.3f ", ring_min/1e6/size, b_eff_rings_msg[i_msg]/1e6/size, ring_max/1e6/size);

      /*worst random, best random, logavg random*/
      random_min = b_eff_pat_msg[i_msg][6];
      random_max = b_eff_pat_msg[i_msg][6];
      for (i_pat=7; i_pat<=35; i_pat++)
      {
        if (b_eff_pat_msg[i_msg][i_pat] < random_min) random_min = b_eff_pat_msg[i_msg][i_pat];
        if (b_eff_pat_msg[i_msg][i_pat] > random_max) random_max = b_eff_pat_msg[i_msg][i_pat];
      }
      fprintf(plot,"%7.3f %7.3f %7.3f ", random_min/1e6/size, b_eff_random_msg[i_msg]/1e6/size, random_max/1e6/size);
      fprintf(plot,"%7.3f ", b_eff_final_msg[i_msg]/1e6/size);
      fprintf(plot,"\n");
    }
	fclose(plot);

  /*--------------------------------------------------------------------------*/
  /* Create result.gps  - script for gnuplot                                  */
  /*                                                                          */
    fprintf(gps,"# script to plot the results of b_eff\n");
    fprintf(gps,"set term postscript eps monochrome  'Times-Roman' 18\n");
    fprintf(gps,"set ylabel 'bandwith [MB/s]'\n");
    fprintf(gps,"set xlabel 'message length per process [Byte]'\n");
    fprintf(gps,"set logscale\n\n");

    for (i=0; i<=3; i++)
    {
      fprintf(gps,"set output 'epsfig%i.eps'\n", 2*i);
      fprintf(gps,"set key right bottom\n");
      fprintf(gps,"set yrange [%s]\n",yrange);
      fprintf(gps,"set title '%s, ring & random patterns'\n", Label[i]);
      fprintf(gps,"plot '%s.plot' index %i using 1:2  title '%s' with linespoints 1 1,\\\n", plotfile, 2*i, pat_description[0]);
      fprintf(gps,"     '%s.plot' index %i using 1:3  title '%s' with linespoints 2 2,\\\n", plotfile, 2*i, pat_description[1]);
      fprintf(gps,"     '%s.plot' index %i using 1:4  title '%s' with linespoints 3 3,\\\n", plotfile, 2*i, pat_description[2]);
      fprintf(gps,"     '%s.plot' index %i using 1:5  title '%s' with linespoints 4 4,\\\n", plotfile, 2*i, pat_description[3]);
      fprintf(gps,"     '%s.plot' index %i using 1:6  title '%s' with linespoints 5 5,\\\n", plotfile, 2*i, pat_description[4]);
      fprintf(gps,"     '%s.plot' index %i using 1:7  title '%s' with linespoints 6 6,\\\n", plotfile, 2*i, pat_description[5]);
      fprintf(gps,"     '%s.plot' index %i using 1:8  title 'worst random' with linespoints 7 7,\\\n", plotfile, 2*i);
      fprintf(gps,"     '%s.plot' index %i using 1:9  title 'avg random' with linespoints 8 8,\\\n", plotfile, 2*i);
      fprintf(gps,"     '%s.plot' index %i using 1:10 title 'best random' with linespoints 9 9\n\n", plotfile, 2*i);
      
      fprintf(gps,"set output 'epsfig%i.eps'\n", 2*i+1);
      fprintf(gps,"set key right bottom\n");
      fprintf(gps,"set yrange [%s]\n",yrange);
      fprintf(gps,"set title '%s, additional patterns'\n", Label[i]);
      fprintf(gps,"plot '%s.plot' index %i using 1:2  title '%s' with linespoints 1 1,\\\n", plotfile, 2*i+1, pat_description[36]);
      fprintf(gps,"     '%s.plot' index %i using 1:3  title '%s' with linespoints 2 2,\\\n", plotfile, 2*i+1, pat_description[37]);
      fprintf(gps,"     '%s.plot' index %i using 1:4  title '%s' with linespoints 3 3,\\\n", plotfile, 2*i+1, pat_description[38]);
      fprintf(gps,"     '%s.plot' index %i using 1:5  title '%s' with linespoints 4 4,\\\n", plotfile, 2*i+1, pat_description[40]);
      fprintf(gps,"     '%s.plot' index %i using 1:6  title '%s' with linespoints 5 5,\\\n", plotfile, 2*i+1, pat_description[41]);
      fprintf(gps,"     '%s.plot' index %i using 1:7  title '%s' with linespoints 6 6,\\\n", plotfile, 2*i+1, pat_description[42]);
      fprintf(gps,"     '%s.plot' index %i using 1:8  title '%s' with linespoints 7 7,\\\n", plotfile, 2*i+1, pat_description[43]);
      fprintf(gps,"     '%s.plot' index %i using 1:9  title '%s' with linespoints 8 8,\\\n", plotfile, 2*i+1, pat_description[44]);
      fprintf(gps,"     '%s.plot' index %i using 1:10 title '%s' with linespoints 9 9\n\n",  plotfile, 2*i+1, pat_description[45]);
    }  
      
    fprintf(gps,"set output 'epsfig8.eps'\n"
                "set key right bottom\n");
    fprintf(gps,"set yrange [%s]\n",yrange);
    fprintf(gps,"set title 'Ring & random average: Sndrcv, Alltoal, non-blk'\n", Label[i]);
    fprintf(gps,"plot '%s.plot' index 8 using 1:2 title 'Sendrcv rings'  with linespoints 1 1,\\\n", plotfile);
    fprintf(gps,"     '%s.plot' index 8 using 1:3 title 'Alltoal rings'  with linespoints 2 2,\\\n", plotfile);
    fprintf(gps,"     '%s.plot' index 8 using 1:4 title 'non-blk rings'  with linespoints 3 3,\\\n", plotfile);
    fprintf(gps,"     '%s.plot' index 8 using 1:5 title 'Sendrcv random' with linespoints 4 4,\\\n", plotfile);
    fprintf(gps,"     '%s.plot' index 8 using 1:6 title 'Alltoal random' with linespoints 5 5,\\\n", plotfile);
    fprintf(gps,"     '%s.plot' index 8 using 1:7 title 'non-blk random' with linespoints 6 6\n\n", plotfile);
     
    fprintf(gps,"set output 'epsfig9.eps'\n"
                "set key right bottom\n");
    fprintf(gps,"set yrange [%s]\n",yrange);
    fprintf(gps,"set title 'Best method: rings & random'\n", Label[i]);
    fprintf(gps,"plot '%s.plot' index 9 using 1:2 title 'rings minumum'         with linespoints 1 1,\\\n", plotfile);
    fprintf(gps,"     '%s.plot' index 9 using 1:3 title 'rings average'         with linespoints 2 2,\\\n", plotfile);
    fprintf(gps,"     '%s.plot' index 9 using 1:4 title 'rings maximum'         with linespoints 3 3,\\\n", plotfile);
    fprintf(gps,"     '%s.plot' index 9 using 1:5 title 'random minimum'        with linespoints 4 4,\\\n", plotfile);
    fprintf(gps,"     '%s.plot' index 9 using 1:6 title 'random average'        with linespoints 5 5,\\\n", plotfile);
    fprintf(gps,"     '%s.plot' index 9 using 1:7 title 'random maximum'        with linespoints 6 6,\\\n", plotfile);
    fprintf(gps,"     '%s.plot' index 9 using 1:8 title 'ring & random average' with linespoints 7 7\n\n", plotfile);
    fclose(gps);

  /*--------------------------------------------------------------------------*/
  /* Create result.tex  - LATEX Source                                        */
  /*                                                                          */
    fprintf(tex,
	"%% Automatically generated by b_eff 3.5\n"
    "%% printing:\n"
    "%% latex result.tex\n"
    "%% dvips -t letter -o result.ps result.dvi\n\n"
    "\\documentclass[]{article}\n"
    "\\usepackage{epsfig}\n\n"
    "\\setlength{\\topmargin}{-15mm}\n"
    "\\setlength{\\textheight}{250mm}\n"
    "\\setlength{\\oddsidemargin}{-10mm}\n"
    "\\setlength{\\textwidth}{180mm}\n\n"
    "\\begin{document}\n"
    "  \\begin{center}\n"
    "    \\Large Effective Bandwith Benchmark ($b_{eff}$) Version 3.5 \\\\[4pt]\n"
    "    \\small High-Performance Computing-Center Stuttgart, HLRS\\\\[0.5cm]\n");
    fprintf(tex,
	"    \\normalsize %s on %s %s %s %s %s\\\\[0.2cm]\n",
			ctime(&t_now), uname_info.sysname, uname_info.nodename, uname_info.release, uname_info.version, uname_info.machine);
	fprintf(tex,
	"    $\\mathbf{b_{eff}}$ \\bfseries = %10.3f MB/s \\mdseries = %7.3f * %3d PEs with %3d MB/PE\\\\[0.1cm]\n", 
			b_eff/1e6, b_eff/1e6/size, size, MEMORY_PER_PROCESSOR);

    fprintf(tex,
	"    \\footnotesize\n"
    "    \\begin{tabular}{c|c|c|c|c|c|c|c|c|c}\n"
    "      & number  & $b_{eff}$ &  Lmax & $b_{eff}$ & $b_{eff}$ & Latency  & Latency  & Latency  & ping-pong \\\\\n"
	"      & of pro- &           &       & at Lmax   & at Lmax   & rings\\&  & rings    & ping-    & bandwidth \\\\\n"
	"      & cessors &           &       & rings\\&   & rings     & random   & only     & pong     &           \\\\\n"
	"      &         &           &       & random    & only      &          &          &          &           \\\\\n"
	"      &         & MByte/s   &       & MByte/s   & MByte/s   & mircosec & microsec & microsec & MByte/s   \\\\\n"
    "    \\hline\n");
    fprintf(tex,
	"    accumulated & %5d & %9.0f & %3.0f MB & %9.0f & %9.0f & %6.3f & %6.3f & %6.3f & %9.0f \\\\[0.2cm]\n",
            size, b_eff/1e6, Lmax/(1024.*1024.), b_eff_final_msg[20]/1e6, b_eff_rings_msg[20]/1e6, 1/(b_eff_final_msg[0]/1e6/size),
            1/(b_eff_rings_msg[0]/1e6/size), latency_onePingPong*1e6, b_asymp_onePingPong/1e6);
    fprintf(tex,
	"    per process & &  %9.0f & %9.0f & %9.0f \\\\ \n",
            b_eff/1e6/size, b_eff_final_msg[20]/1e6/size, b_eff_rings_msg[20]/1e6/size);
	fprintf(tex,
	"    \\end{tabular}\n");
    fprintf(tex,
	"    \\\\[0.1cm]\n"
	"    Ping-Pong result (only the processes with rank 0 and 1 in MPI\\_COMM\\_WORLD were used):\\\\\n"
	"    Latency: %8.3f microsec per message \\hspace{0.5cm} Bandwidth: %10.3f MB/s (with MB/s = $10^6$ byte/s)\n",
			latency_onePingPong*1e6, b_asymp_onePingPong/1e6);
    fprintf(tex,
	"  \\end{center}\n"    
    "  \\begin{figure}[ht]\n"
    "    \\begin{center}\n"
    "      \\epsfig{file=epsfig0.eps}\n"
    "      \\epsfig{file=epsfig1.eps}\n"
    "    \\end{center}\n"
    "  \\end{figure}\n"
    "  \\begin{figure}[ht]\n"
    "    \\begin{center}\n"
    "      \\epsfig{file=epsfig2.eps}\n"
    "      \\epsfig{file=epsfig3.eps}\n"
    "    \\end{center}\n"
    "  \\end{figure}\n"
    "  \\begin{figure}[ht]\n"
    "    \\begin{center}\n"
    "      \\epsfig{file=epsfig4.eps}\n"
    "      \\epsfig{file=epsfig5.eps}\n"
    "    \\end{center}\n"
    "  \\end{figure}\n"
    "  \\begin{figure}[ht]\n"
    "    \\begin{center}\n"
    "      \\epsfig{file=epsfig6.eps}\n"
    "      \\epsfig{file=epsfig7.eps}\n"
    "    \\end{center}\n"
    "  \\end{figure}\n"
    "  \\begin{figure}[ht]\n"
    "    \\begin{center}\n"
    "      \\epsfig{file=epsfig8.eps}\n"
    "      \\epsfig{file=epsfig9.eps}\n"
    "    \\end{center}\n"
    "  \\end{figure}\n"
    "\\end{document}\n");
    fclose(tex);

#   endif
# endif
  }
 
#ifdef _OPENMP
  } /*end of master thread block*/ 
#pragma omp barrier 
 } /*end of parallel block*/ 
#endif 

  MPI_Finalize();
  return(0); 
}
